Kaggle competition: [Student] Shopee Code League - Product Detection
<SOS> metric: categorizationaccuracy Kaggle data source: [student]-shopee-code-league-product-detection<set_options>
!pip install -q --upgrade efficientnet tensorflow_addons
_ = gc.collect()<merge>
print(f'Numpy version : {np.__version__}')
 print(f'Tensorflow version : {tf.__version__}')
 print(f'Tensorflow Addons version : {tfa.__version__}')
 print(f'EfficientNet(library)version : {efficientnet.__version__}')
 print(f'Matplotlib version : {matplotlib.__version__}')
 print(f'Scipy version : {scipy.__version__}')
 print(f'Pandas version : {pd.__version__}')
 print(f'Scikit-Learn version : {sklearn.__version__}' )
logger.info("join content features to validation...")
 valid = fast_left_join(
 valid,
content_df,
on="content_id"
)<data_type_conversions>
PRE_TRAINING_TIME_START = datetime.now()
AUTO = tf.data.experimental.AUTOTUNE
SEED = 42
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 os.environ['TF_DETERMINISTIC_OPS']=str(SEED)
 tf.random.set_seed(SEED)
train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean )<define_variables>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
logger.info("generating questions features...")
 question_df = questions[[
"question_id", "part"
]].set_index("question_id" )<merge>
IMAGE_SIZE =(512, 512)
 GCS_TRAIN_PATHS = [
KaggleDatasets().get_gcs_path('tfrecords'),
KaggleDatasets().get_gcs_path('tfrecords-2')
 ]
TRAINING_FILENAMES = []
for i in GCS_TRAIN_PATHS:
TRAINING_FILENAMES.append(tf.io.gfile.glob(i + '/*.tfrecords'))
TRAINING_FILENAMES = list(itertools.chain.from_iterable(TRAINING_FILENAMES))
GCS_TEST_PATH = KaggleDatasets().get_gcs_path('tfrecords-3')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_TEST_PATH + '/*.tfrecords')
 print(len(TRAINING_FILENAMES))
print(len(TEST_FILENAMES))
logger.info("joining questions features to validation...")
 valid = fast_left_join(
 valid,
question_df,
left_on = "content_id",
right_on = "question_id"
)<merge>
EPOCHS = 12
DO_AUG = True
BATCH_SIZE = 256
current_epoch = 0
chance = 0
NUM_TRAINING_IMAGES = 105390
NUM_TEST_IMAGES = 12186
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
logger.info("joining questions features to train...")
 train = fast_left_join(
 train,
question_df,
left_on = "content_id",
right_on = "question_id"
)<data_type_conversions>
CLASSES = [str(c ).zfill(2)for c in range(0, 42)]
logger.info("filling missing values...")
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )<import_modules>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"label": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
label = tf.cast(example['label'], tf.int32)
 return(( image, words), label)
 def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"filename": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
filename = example['filename']
return(( image, words), filename)
 def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset(do_aug=True):
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 if do_aug:
dataset = dataset.map(image_augmentation, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, tta=None):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 if tta == 0:
dataset = dataset.map(image_augmentation_tta_0, num_parallel_calls=AUTO)
 elif tta == 1:
dataset = dataset.map(image_augmentation_tta_1, num_parallel_calls=AUTO)
 elif tta == 2:
dataset = dataset.map(image_augmentation_tta_2, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
import lightgbm as lgb<define_variables>
@tf.function
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
@tf.function
def transform(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
rot = 17.* tf.random.normal([1],dtype='float32')
 else:
rot = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
shr = 5.5 * tf.random.normal([1],dtype='float32')
 else:
shr = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_zoom = tf.random.normal([1],dtype='float32')/8.5
if h_zoom > 0:
h_zoom = 1.0 + h_zoom * -1
else:
h_zoom = 1.0 + h_zoom
else:
h_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_zoom = tf.random.normal([1],dtype='float32')/8.5
if w_zoom > 0:
w_zoom = 1.0 + w_zoom * -1
else:
w_zoom = 1.0 + w_zoom
else:
w_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
h_shift = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
w_shift = tf.constant([0],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3] )
TARGET = "answered_correctly"
FEATS = [
"correct_sum",
"answer_cnt",
"acc",
"part",
"content_acc",
"prior_question_elapsed_time",
"prior_question_had_explanation"
]<prepare_x_and_y>
@tf.function
def transform_grid_mark(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + tf.cast(w, tf.float32)//2.) , tf.round(old_coords[1, :] + tf.cast(h, tf.float32)//2.)
old_coords_x = tf.cast(old_coords_x, tf.int32)
 old_coords_y = tf.cast(old_coords_y, tf.int32)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 @tf.function
def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform_grid_mark(image, rot_mat_inv, image_shape)
 @tf.function
def grid_mask() :
h = tf.constant(IMAGE_SIZE[0], dtype=tf.float32)
 w = tf.constant(IMAGE_SIZE[1], dtype=tf.float32)
 image_height, image_width =(h, w)
 d1 = 112
d2 = 352
rotate_angle = 45
ratio = 0.6
hh = tf.math.ceil(tf.math.sqrt(h*h+w*w))
hh = tf.cast(hh, tf.int32)
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-tf.cast(h, tf.int32)) //2,(hh-tf.cast(w, tf.int32)) //2, tf.cast(image_height, tf.int32), tf.cast(image_width, tf.int32))
return mask
@tf.function
def apply_grid_mask(image):
mask = grid_mask()
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, 'float32' )
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _ = gc.collect()<create_dataframe>
@tf.function
def image_augmentation_tta_0(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_1(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_2(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 if tf.random.uniform(shape=[], minval=0, maxval=10, dtype=tf.int32, seed=SEED)< max_chance:
image = apply_grid_mask(image)
 return(( image, words), filename )
lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature=["part"])
 lgb_valid = lgb.Dataset(valid[FEATS], y_va, reference=lgb_train, categorical_feature=["part"])
 del train, y_tr
_=gc.collect()<import_modules>
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, AveragePooling2D, GlobalAveragePooling2D, SpatialDropout2D, BatchNormalization, Activation, Concatenate
from sklearn.metrics import roc_auc_score<train_model>
class EpochCallback(tf.keras.callbacks.Callback):
def on_epoch_begin(self, epoch, logs=None):
global current_epoch
global chance
current_epoch = epoch
if current_epoch < 2:
chance = 0
elif current_epoch < 9:
chance = current_epoch - 1
else:
chance = 8
print(f'Epoch
print(datetime.now() )
USE_OFFLINE_MODEL = True
model_path = os.path.join(processed_data_dir, "model_7fea.pkl")
 if USE_OFFLINE_MODEL:
logger.info("loading offline trained model...")
 model = pickle_load(model_path)
 else:
logger.info("start training model...")
 model = lgb.train(
 {
"objective": "binary",
"metric": "auc",
"boosting_type": "gbdt",
"learning_rate": 0.3,
"seed": 3
},
lgb_train,
valid_sets=[lgb_train, lgb_valid],
verbose_eval=20,
num_boost_round=1000,
early_stopping_rounds=10 
)
 auc = roc_auc_score(y_va, model.predict(valid[FEATS]))
logger.info(f"auc on validation: {auc}" )<set_options>
es_val_acc = tf.keras.callbacks.EarlyStopping(
 monitor='val_sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_val_loss = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_acc = tf.keras.callbacks.EarlyStopping(
 monitor='sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 es_loss = tf.keras.callbacks.EarlyStopping(
 monitor='loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 epoch_cb = EpochCallback()
del lgb_train, lgb_valid
_ = gc.collect()<predict_on_test>
with strategy.scope() :
efn7 = efn.EfficientNetB7(weights='noisy-student', include_top=False, input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
for layer in efn7.layers:
layer.trainable = True
model_image = Sequential([
efn7,
GlobalAveragePooling2D(name='efficientnet-b7_gap'),
], name='b7-image')
 model_words = Sequential([
Input(( 6633,), name='mlp-words_input'),
Dense(331, name='mlp-words_dense_1'),
BatchNormalization(name='mlp-words_bn_1'),
Activation('relu', name='mlp-words_act_1'),
Dense(110, name='mlp-words_dense_2'),
BatchNormalization(name='mlp-words_bn_2'),
Activation('relu', name='mlp-words_act_2'),
], name='mlp-words')
 concatenate = Concatenate(name='concatenate' )([model_image.output, model_words.output])
 output = Dense(len(CLASSES), activation='softmax', name='output' )(concatenate)
 model = Model(inputs=[model_image.input, model_words.input], outputs=output)
 model.compile(optimizer=tfa.optimizers.LAMB(0.01), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 model.summary()
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<import_modules>
print(f'Pre training time : {(datetime.now() - PRE_TRAINING_TIME_START ).total_seconds() } seconds' )
import time<categorify>
LR_START = 0.0005
LR_MAX = 0.001
LR_MIN = 0.00015
LR_RAMPUP_EPOCHS = 2
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.83
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 plt_lr(EPOCHS )
pre_test_df = None
infer_start, iter_cnt = time.time() , 0
actuals = []
preds = []
for(test_df, sample_pred_df)in iter_test:
if pre_test_df is not None:
targets = eval(test_df["prior_group_answers_correct"].iloc[0])
 actuals.extend(targets)
 pre_test_df[TARGET] = targets
update_user_feats(pre_test_df, correct_sum, answer_cnt)
 pre_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats(
 test_df, correct_sum, answer_cnt, update=False 
)
 test_df = fast_left_join(
 test_df, content_df, on="content_id" 
)
 test_df = fast_left_join(
 test_df, question_df, left_on="content_id", right_on="question_id" 
)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 preds_ = model.predict(test_df[FEATS])
 test_df[TARGET] = preds_
preds.extend(preds_.tolist())
 set_predict(test_df[['row_id', TARGET]])
 iter_cnt += 1
infer_duration = time.time() - infer_start
logger.info(f"total iterations: {iter_cnt}, cost {infer_duration:.4f} secs, {infer_duration/iter_cnt:.4f} secs/iteration" )<compute_test_metric>
model.fit(
 get_training_dataset(do_aug=DO_AUG), steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[es_acc, epoch_cb, lr_schedule], verbose=1
)
roc_auc_score(actuals, preds[:len(actuals)] )<import_modules>
h = model.history
plt_acc(h)
 plt_loss(h )
base_dir = '/kaggle/input/siim-isic-melanoma-classification/'
external_base_dir = '/kaggle/input/melanoma-external-malignant-256/'
external_tfrec_base_dir = '.. /input/melanoma-256x256'
sns.set(style="darkgrid")
 random.seed(42)
<define_variables>
POST_TRAINING_TIME_START = datetime.now()
DEVICE = 'GPU'<choose_model_class>
def test(tta=None):
test_ds = get_test_dataset(ordered=True, tta=tta)
 print(f'Computing predictions for TTA {tta}...')
 test_images_ds = test_ds.map(lambda iw, filename: [iw])
 model_pred = model.predict(test_images_ds)
 return model_pred
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
model_pred = test(tta=None)
 model_pred_tta_0 = test(tta=0)
 model_pred_tta_1 = test(tta=1)
 model_pred_tta_2 = test(tta=2 )
IMG_HEIGHT = 256
IMG_WIDTH = 256
N_CHANNELS = 3
epochs = 20
BATCH_SIZE = 8 * REPLICAS
IMAGE_SIZE = [IMG_HEIGHT, IMG_WIDTH]
IMAGE_RESIZE = [IMG_HEIGHT, IMG_WIDTH]
input_shape =(IMG_HEIGHT, IMG_WIDTH, N_CHANNELS)
 BALANCE_DATA = True
aug_data = True
NETWORK_MODEL = 'DenseNet201'<define_variables>
pred_plain = np.argmax(model_pred, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_plain.shape[0])) ).numpy().astype('U')
 df_submission = pd.DataFrame({'filename': test_ids, 'category': pred_plain})
 df_submission = df_submission.drop_duplicates()
df_submission['category'] = df_submission['category'].apply(lambda c: str(c ).zfill(2))
df_submission
if DEVICE == 'TPU':
files_dir = KaggleDatasets().get_gcs_path('melanoma-256x256')
 else:
files_dir = external_tfrec_base_dir<load_from_csv>
df_submission.to_csv('submission.csv', index=False)
 !head submission.csv
TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(
 tf.io.gfile.glob(files_dir + '/train*.tfrec'),
test_size=0.1, random_state=42 
)
 TEST_FILENAMES = tf.io.gfile.glob(files_dir + '/test*.tfrec')
 print('Train TFRecord Files:', len(TRAINING_FILENAMES))
print('Validation TFRecord Files:', len(VALID_FILENAMES))
print('Test TFRecord Files:', len(TEST_FILENAMES))
submission_example = pd.read_csv(base_dir + 'sample_submission.csv' )<normalization>
pred_tta_0 = np.mean(np.array([model_pred, model_pred_tta_0]), axis=0)
 pred_tta_0 = np.argmax(pred_tta_0, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_0.shape[0])) ).numpy().astype('U')
 df_submission_tta_0 = pd.DataFrame({'filename': test_ids, 'category': pred_tta_0})
 df_submission_tta_0 = df_submission_tta_0.drop_duplicates()
df_submission_tta_0['category'] = df_submission_tta_0['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))<categorify>
df_submission_tta_0.to_csv('submission_tta_0.csv', index=False)
 !head submission_tta_0.csv
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 2.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 h_shift = 8.* tf.random.normal([1],dtype='float32')
 w_shift = 8.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<normalization>
pred_tta_all = np.mean(np.array([model_pred, model_pred_tta_0, model_pred_tta_1, model_pred_tta_2]), axis=0)
 pred_tta_all = np.argmax(pred_tta_all, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_all.shape[0])) ).numpy().astype('U')
 df_submission_tta_all = pd.DataFrame({'filename': test_ids, 'category': pred_tta_all})
 df_submission_tta_all = df_submission.drop_duplicates()
df_submission_tta_all['category'] = df_submission_tta_all['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_all
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH])
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [IMG_HEIGHT, IMG_WIDTH, 3])
 return image
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTO)
 return dataset
def augmentation_pipeline(image, label):

 if aug_data:
image,_ = transform(image,label)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<create_dataframe>
df_submission_tta_all.to_csv('submission_tta_all.csv', index=False)
 !head submission_tta_all.csv
train_dataset = get_training_dataset()
valid_dataset = get_validation_dataset()<define_variables>
model.save('model.h5')
 model.save_weights('model_weights.h5' )
<prepare_x_and_y><EOS> 
print(f'Post training time : {(datetime.now() - POST_TRAINING_TIME_START ).total_seconds() } seconds' )