Kaggle competition: [Student] Shopee Code League - Product Detection
<SOS> metric: categorizationaccuracy Kaggle data source: [student]-shopee-code-league-product-detection<create_dataframe>
!pip install --upgrade efficientnet tensorflow_addons tensorflow
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
user_performance = pd.DataFrame(tmp['answered_correctly'] - tmp['question_average'], columns=['performance'])
 del tmp 
<merge>
import math
import re
import random
import os
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
import efficientnet
import itertools
import matplotlib
import scipy
import pandas as pd
import sklearn
from matplotlib import pyplot as plt
from datetime import datetime
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']
del user_performance, user_count, user_average
gc.collect()<groupby>
print(f'Numpy version : {np.__version__}')
 print(f'Tensorflow version : {tf.__version__}')
 print(f'Tensorflow Addons version : {tfa.__version__}')
 print(f'EfficientNet(library)version : {efficientnet.__version__}')
 print(f'Matplotlib version : {matplotlib.__version__}')
 print(f'Scipy version : {scipy.__version__}')
 print(f'Pandas version : {pd.__version__}')
 print(f'Scikit-Learn version : {sklearn.__version__}' )
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )<prepare_output>
!pip freeze > requirements.txt
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_<create_dataframe>
PRE_TRAINING_TIME_START = datetime.now()
AUTO = tf.data.experimental.AUTOTUNE
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df<create_dataframe>
SEED = 42
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 os.environ['TF_DETERMINISTIC_OPS']=str(SEED)
 tf.random.set_seed(SEED)
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df<create_dataframe>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )<define_variables>
IMAGE_SIZE =(512, 512)
 GCS_TRAIN_PATHS = [
KaggleDatasets().get_gcs_path('tfrecords'),
KaggleDatasets().get_gcs_path('tfrecords-2')
 ]
TRAINING_FILENAMES = []
for i in GCS_TRAIN_PATHS:
TRAINING_FILENAMES.append(tf.io.gfile.glob(i + '/*.tfrecords'))
TRAINING_FILENAMES = list(itertools.chain.from_iterable(TRAINING_FILENAMES))
GCS_TEST_PATH = KaggleDatasets().get_gcs_path('tfrecords-3')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_TEST_PATH + '/*.tfrecords')
 print(len(TRAINING_FILENAMES))
print(len(TEST_FILENAMES))
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average']<drop_column>
EPOCHS = 12
DO_AUG = True
BATCH_SIZE = 256
current_epoch = 0
chance = 0
NUM_TRAINING_IMAGES = 105390
NUM_TEST_IMAGES = 12186
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
train = train[['row_id','user_id','content_id', 'content_type_id', 'answered_correctly']].reset_index()<merge>
CLASSES = [str(c ).zfill(2)for c in range(0, 42)]
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df<categorify>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"label": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
label = tf.cast(example['label'], tf.int32)
 return(( image, words), label)
 def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"filename": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
filename = example['filename']
return(( image, words), filename)
 def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset(do_aug=True):
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 if do_aug:
dataset = dataset.map(image_augmentation, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, tta=None):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 if tta == 0:
dataset = dataset.map(image_augmentation_tta_0, num_parallel_calls=AUTO)
 elif tta == 1:
dataset = dataset.map(image_augmentation_tta_1, num_parallel_calls=AUTO)
 elif tta == 2:
dataset = dataset.map(image_augmentation_tta_2, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
%%time
train = data_transform(train )<categorify>
@tf.function
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
@tf.function
def transform(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
rot = 17.* tf.random.normal([1],dtype='float32')
 else:
rot = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
shr = 5.5 * tf.random.normal([1],dtype='float32')
 else:
shr = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_zoom = tf.random.normal([1],dtype='float32')/8.5
if h_zoom > 0:
h_zoom = 1.0 + h_zoom * -1
else:
h_zoom = 1.0 + h_zoom
else:
h_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_zoom = tf.random.normal([1],dtype='float32')/8.5
if w_zoom > 0:
w_zoom = 1.0 + w_zoom * -1
else:
w_zoom = 1.0 + w_zoom
else:
w_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
h_shift = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
w_shift = tf.constant([0],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3] )
%%time
test = data_transform(test,False,True )<prepare_x_and_y>
@tf.function
def transform_grid_mark(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + tf.cast(w, tf.float32)//2.) , tf.round(old_coords[1, :] + tf.cast(h, tf.float32)//2.)
old_coords_x = tf.cast(old_coords_x, tf.int32)
 old_coords_y = tf.cast(old_coords_y, tf.int32)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 @tf.function
def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform_grid_mark(image, rot_mat_inv, image_shape)
 @tf.function
def grid_mask() :
h = tf.constant(IMAGE_SIZE[0], dtype=tf.float32)
 w = tf.constant(IMAGE_SIZE[1], dtype=tf.float32)
 image_height, image_width =(h, w)
 d1 = 112
d2 = 352
rotate_angle = 45
ratio = 0.6
hh = tf.math.ceil(tf.math.sqrt(h*h+w*w))
hh = tf.cast(hh, tf.int32)
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-tf.cast(h, tf.int32)) //2,(hh-tf.cast(w, tf.int32)) //2, tf.cast(image_height, tf.int32), tf.cast(image_width, tf.int32))
return mask
@tf.function
def apply_grid_mask(image):
mask = grid_mask()
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, 'float32' )
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]<create_dataframe>
@tf.function
def image_augmentation_tta_0(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_1(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_2(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 if tf.random.uniform(shape=[], minval=0, maxval=10, dtype=tf.int32, seed=SEED)< max_chance:
image = apply_grid_mask(image)
 return(( image, words), filename )
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1000,
'num_leaves': 80,
'num_iterations' : 100
}
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )<set_options>
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, AveragePooling2D, GlobalAveragePooling2D, SpatialDropout2D, BatchNormalization, Activation, Concatenate
del X_train,y_train
gc.collect()<train_model>
class EpochCallback(tf.keras.callbacks.Callback):
def on_epoch_begin(self, epoch, logs=None):
global current_epoch
global chance
current_epoch = epoch
if current_epoch < 2:
chance = 0
elif current_epoch < 9:
chance = current_epoch - 1
else:
chance = 8
print(f'Epoch
print(datetime.now() )
model = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train,lgb_val],
verbose_eval=1,
num_boost_round=100,
early_stopping_rounds=10
)<predict_on_test>
es_val_acc = tf.keras.callbacks.EarlyStopping(
 monitor='val_sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_val_loss = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_acc = tf.keras.callbacks.EarlyStopping(
 monitor='sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 es_loss = tf.keras.callbacks.EarlyStopping(
 monitor='loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 epoch_cb = EpochCallback()
%%time
predictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index )<compute_test_metric>
with strategy.scope() :
efn7 = efn.EfficientNetB7(weights='noisy-student', include_top=False, input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
for layer in efn7.layers:
layer.trainable = True
model_image = Sequential([
efn7,
GlobalAveragePooling2D(name='efficientnet-b7_gap'),
], name='b7-image')
 model_words = Sequential([
Input(( 6633,), name='mlp-words_input'),
Dense(331, name='mlp-words_dense_1'),
BatchNormalization(name='mlp-words_bn_1'),
Activation('relu', name='mlp-words_act_1'),
Dense(110, name='mlp-words_dense_2'),
BatchNormalization(name='mlp-words_bn_2'),
Activation('relu', name='mlp-words_act_2'),
], name='mlp-words')
 concatenate = Concatenate(name='concatenate' )([model_image.output, model_words.output])
 output = Dense(len(CLASSES), activation='softmax', name='output' )(concatenate)
 model = Model(inputs=[model_image.input, model_words.input], outputs=output)
 model.compile(optimizer=tfa.optimizers.LAMB(0.01), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 model.summary()
roc_auc_score(y_test,predictions[0] )<predict_on_test>
print(f'Pre training time : {(datetime.now() - PRE_TRAINING_TIME_START ).total_seconds() } seconds' )
%%time
env = riiideducation.make_env()
iter_test = env.iter_test()
iter_nb = 0
for(current_df, sample_prediction_df)in iter_test:
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df)
 prior_df = current_df.copy()
current_df = data_transform(current_df,False,False)
 current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])
 iter_nb = 1
env.predict(current_df.loc[:, ['row_id', 'answered_correctly']] )<define_variables>
LR_START = 0.0005
LR_MAX = 0.001
LR_MIN = 0.00015
LR_RAMPUP_EPOCHS = 2
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.83
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 plt_lr(EPOCHS )
RUN_ON_KAGGLE = True
RANDOM_SEED = 3
random.seed(RANDOM_SEED)
 np.random.seed(seed=RANDOM_SEED )<define_variables>
model.fit(
 get_training_dataset(do_aug=DO_AUG), steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[es_acc, epoch_cb, lr_schedule], verbose=1
)
processed_data_dir = "/kaggle/input/riid-acp-data-joined-user-feat"<load_pretrained>
h = model.history
plt_acc(h)
 plt_loss(h )
logger.info("reading training data...")
 train = pd.read_pickle(
 os.path.join(processed_data_dir, "train-with-user-fea.pkl")
 )<load_pretrained>
POST_TRAINING_TIME_START = datetime.now()
logger.info("reading validation data...")
 valid = pd.read_pickle(
 os.path.join(processed_data_dir, "eval-with-user-fea.pkl")
 )<drop_column>
def test(tta=None):
test_ds = get_test_dataset(ordered=True, tta=tta)
 print(f'Computing predictions for TTA {tta}...')
 test_images_ds = test_ds.map(lambda iw, filename: [iw])
 model_pred = model.predict(test_images_ds)
 return model_pred
train.drop("row_id", inplace=True, axis=1)
 valid.drop("row_id", inplace=True, axis=1 )<load_pretrained>
model_pred = test(tta=None)
 model_pred_tta_0 = test(tta=0)
 model_pred_tta_1 = test(tta=1)
 model_pred_tta_2 = test(tta=2 )
def pickle_load(file_path):
with open(file_path, "rb")as f:
return pickle.load(f )<load_pretrained>
pred_plain = np.argmax(model_pred, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_plain.shape[0])) ).numpy().astype('U')
 df_submission = pd.DataFrame({'filename': test_ids, 'category': pred_plain})
 df_submission = df_submission.drop_duplicates()
df_submission['category'] = df_submission['category'].apply(lambda c: str(c ).zfill(2))
df_submission
answer_cnt = defaultdict(int)
 answer_cnt.update(
 pickle_load(os.path.join(processed_data_dir, "answer_cnt.pkl")) 
)
 correct_sum = defaultdict(int)
 correct_sum.update(
 pickle_load(os.path.join(processed_data_dir, "correct_sum.pkl"))
)<load_from_csv>
df_submission.to_csv('submission.csv', index=False)
 !head submission.csv
logger.info("reading questions...")
 questions = pd.read_csv(
 os.path.join(data_folder, "questions.csv"),
dtype={
"question_id": "int16",
"bundle_id": "int32",
"correct_answer": "int8",
"part": "int8"
}
)<load_from_csv>
pred_tta_0 = np.mean(np.array([model_pred, model_pred_tta_0]), axis=0)
 pred_tta_0 = np.argmax(pred_tta_0, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_0.shape[0])) ).numpy().astype('U')
 df_submission_tta_0 = pd.DataFrame({'filename': test_ids, 'category': pred_tta_0})
 df_submission_tta_0 = df_submission_tta_0.drop_duplicates()
df_submission_tta_0['category'] = df_submission_tta_0['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_0
logger.info("reading lectures...")
 lectures = pd.read_csv(
 os.path.join(data_folder, "lectures.csv"),
dtype={
"lecture_id": "int16",
"tag": "int16",
"part": "int8"
}
)<data_type_conversions>
df_submission_tta_0.to_csv('submission_tta_0.csv', index=False)
 !head submission_tta_0.csv
def add_user_feats(
 df,
user_correct_sum: dict,
user_answer_cnt: dict,
update=True 
):
correct_sum = np.zeros(len(df), dtype=np.int32)
 answer_cnt = np.zeros(len(df), dtype=np.int32)
 cols = ["user_id", "answered_correctly"] if update else ["user_id"]
for i, row in df[cols].iterrows() :
user_id = row["user_id"]
correct_sum[i] = user_correct_sum[user_id]
answer_cnt[i] = user_answer_cnt[user_id]
if update:
answer_res = row["answered_correctly"]
user_correct_sum[user_id] += answer_res
user_answer_cnt[user_id] += 1
user_feats = pd.DataFrame({
"correct_sum": correct_sum,
"answer_cnt": answer_cnt,
"acc":(correct_sum / answer_cnt ).astype(np.float32)
 })
 return pd.concat([df, user_feats], axis=1 )<feature_engineering>
pred_tta_all = np.mean(np.array([model_pred, model_pred_tta_0, model_pred_tta_1, model_pred_tta_2]), axis=0)
 pred_tta_all = np.argmax(pred_tta_all, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_all.shape[0])) ).numpy().astype('U')
 df_submission_tta_all = pd.DataFrame({'filename': test_ids, 'category': pred_tta_all})
 df_submission_tta_all = df_submission.drop_duplicates()
df_submission_tta_all['category'] = df_submission_tta_all['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_all
def update_user_feats(df, user_correct_sum: dict, user_answer_cnt: dict):
df = df[df["content_type_id"] == 0]
for _, row in df[["user_id", "answered_correctly"]].iterrows() :
user_id = row["user_id"]
answer_res = row["answered_correctly"]
user_correct_sum[user_id] += answer_res
user_answer_cnt[user_id] += 1<concatenate>
df_submission_tta_all.to_csv('submission_tta_all.csv', index=False)
 !head submission_tta_all.csv
def fast_left_join(left, right, on=None, left_on=None, right_on=None):
left_on = left_on or on
right_on = right_on or on
assert left_on in left.columns, f"{on} not found in left DF"
assert right_on == right.index.name, f"{right_on} not equals to right's index_name"
return pd.concat([
left.reset_index(drop=True),
right.reindex(left[left_on].values ).reset_index(drop=True)
 ], axis=1 )<data_type_conversions>
model.save('model.h5')
 model.save_weights('model_weights.h5' )
<merge><EOS> 
print(f'Post training time : {(datetime.now() - POST_TRAINING_TIME_START ).total_seconds() } seconds' )