Kaggle competition: iMet Collection 2019 - FGVC6
if TUNER_ON is True:
tuner = RandomSearch(build_model,
objective='val_loss',
max_trials=18,
executions_per_trial=1,
)<split>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 156
NUM_CLASSES = 1103
beta_f2=2
tf.config.run_functions_eagerly(True)
 epochs = 5
batch_size = 64
es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
 X_train_pure, X_valid, y_train_pure, y_valid = train_test_split(train_x_new, train_y, test_size = 0.15, random_state=0)
 ohe_train, ohe_valid = train_test_split(ohe_df, test_size = 0.15, random_state=0)
 ohe_location_train, ohe_location_valid = train_test_split(ohe_location_df, test_size = 0.15, random_state=0)
 print(ohe_train.shape)
 print(ohe_location_train.shape)
 meta_train = pd.concat([ohe_train, ohe_location_train], axis=1, sort=False)
 meta_valid = pd.concat([ohe_valid, ohe_location_valid], axis=1, sort=False)
 if TUNER_ON is True:
tuner.search([X_train_pure, meta_train], y_train_pure, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid))
best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]
model_real = tuner.hypermodel.build(best_hps)
 else:
print('Using KerasClassifier method to select best hyperparameters for model')
 clf = KerasClassifier(build_fn=build_model_wrapper)
 parameters = {
'epochs': [10],
'batch_size':[32],
'dense_size':[32],
'optimizer':['Adam'],
'dropout_value': [0.1]
}
kfold_splits = 2
grid_search = GridSearchCV(estimator=clf,
n_jobs=-1,
verbose=1,
return_train_score=True,
cv=kfold_splits,
param_grid=parameters,)
 combi_input = np.concatenate([X_train_pure, meta_train], axis=1)
 if DASK_ON is True:
with joblib.parallel_backend('dask'):
grid_search.fit(combi_input, y_train_pure)
 else:
grid_search.fit(combi_input, y_train_pure)
 model_real = grid_search.best_estimator_
print(grid_search.best_parameters_)
 model_real.summary()<train_model>
path_to_train = '.. /input/imet-2019-fgvc6/train/'
data = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_dataset_info = []
for name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info )
if DASK_ON is True:
with joblib.parallel_backend('dask'):
fit_history = model_real.fit([X_train_pure, meta_train], y_train_pure, batch_size = batch_size, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid), verbose=1)
 else:
fit_history = model_real.fit([X_train_pure, meta_train], y_train_pure, batch_size = batch_size, epochs=epochs, callbacks=[es_callback], validation_data =([X_valid, meta_valid], y_valid), verbose=1 )<load_from_csv>
gamma = 2.0
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = K.clip(pt, epsilon, 1-epsilon)
 CE = -K.log(pt)
 FL = K.pow(1-pt, gamma)* CE
loss = K.sum(FL, axis=1)
 return loss
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test_df.shape<save_to_csv>
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 class data_generator(Sequence):
def create_train(dataset_info, batch_size, shape, augument=True):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(image/255.)
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def create_valid(dataset_info, batch_size, shape, augument=False):
assert shape[2] == 3
while True:
for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(image/255.)
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, shape):
image = cv2.imread(path+'.png')
 image = cv2.resize(image,(SIZE, SIZE))
return image
def augment(image):
augment_img = iaa.Sequential([
iaa.OneOf([
iaa.Affine(rotate=0),
iaa.Affine(rotate=(-15,15)) ,
iaa.Crop(px=(0, 16)) ,
iaa.Affine(shear=(-5, 5)) ,
iaa.GaussianBlur(sigma=(0, 0.5)) ,
iaa.Fliplr(0.5),
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug
test_df['keyword_cleaned'] = test_df['keyword'].map(keyword_cleaner)
 transformed = keyword_enc.transform(test_df['keyword_cleaned'].to_numpy().reshape(-1, 1))
ohe_df = pd.DataFrame(transformed.todense() , columns=keyword_enc.get_feature_names())
 ohe_df.to_csv('ohe_test_df.csv', index=False )<save_to_csv>
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential, load_model
from keras.layers import *
from keras.applications import *
from keras.callbacks import ModelCheckpoint
from keras import metrics
from keras.optimizers import Adam
from keras import backend as K
import keras
from keras.models import Model
test_df['location_cleaned'] = test_df['location'].map(location_cleaner)
 transformed = location_enc.transform(test_df['location_cleaned'].to_numpy().reshape(-1, 1))
ohe_location_df = pd.DataFrame(transformed.todense() , columns=location_enc.get_feature_names())
 ohe_location_df.to_csv('ohe_location_test_df.csv', index=False )<string_transform>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = Xception(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights('.. /input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
X_test = test_df[['text']]
test_sequences = []
for snippet in X_test['text'].tolist() :
sequences = []
doc = nlp(snippet)
 for token in doc:
lower_token = token.text.lower()
if(lower_token not in word_index):
sequences.append(word_index['<OOV>'])
 else:
sequences.append(word_index[lower_token])
 test_sequences.append(sequences)
 X_test_new = pad_sequences(test_sequences, maxlen=max_len)
 meta_test = pd.concat([ohe_df, ohe_location_df], axis=1, sort=False)
 preds = model_real.predict([X_test_new, meta_test])
 preds = np.ndarray.round(preds ).astype(int)
 preds<save_to_csv>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 35; batch_size = 64
checkpoint = ModelCheckpoint('.. /working/Resnet50_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 indexes = np.arange(train_dataset_info.shape[0])
 train_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)
 train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=True)
 train_generator_warmup = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 validation_generator = data_generator.create_valid(
 train_dataset_info[valid_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),
epochs=1, batch_size=batch_size, verbose=0)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
sub_df = pd.DataFrame({'id': test_df.id, 'target': preds.flatten() })
 sub_df.to_csv('submission21.csv', index=False )<set_options>
model = create_model(
 input_shape=(SIZE,SIZE,3),
n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-5,0):
model.layers[i].trainable = True
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(1e-3))
model.fit_generator(
 train_generator_warmup,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
epochs=2,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
verbose=1 )
plt.style.use('ggplot')
<load_from_csv>
for layer in model.layers:
layer.trainable = True
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.0003))
checkpoint = ModelCheckpoint('.. /working/Resnet50_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=(epochs*0.9),
verbose=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list)
 model.compile(loss='binary_crossentropy',
optimizer=SGD(0.0001,0.9))
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=(epochs*0.1),
verbose=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 display(train.head())
 print(len(train))
display(test.head())
 print(len(test))<string_transform>
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 model.load_weights('.. /working/Resnet50_focal.h5')
 predicted = []
def create_corpus(target):
corpus = []
for x in train[train["target"] == target]["text"].str.split() :
print(x)
 for i in x:
corpus.append(i)
 return corpus<import_modules>

 BATCH = 512
fullValGen = data_generator.create_valid(
 train_dataset_info[valid_indexes], BATCH,(SIZE,SIZE,3))
n_val = round(train_dataset_info.shape[0]*0.15)//BATCH
print(n_val)
 lastFullValPred = np.empty(( 0, NUM_CLASSES))
lastFullValLabels = np.empty(( 0, NUM_CLASSES))
for i in tqdm(range(n_val+1)) :
im, lbl = next(fullValGen)
 scores = model.predict(im)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )
from collections import defaultdict
from nltk.corpus import stopwords
from nltk.util import ngrams<feature_engineering>
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return f2
def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score
corpus = create_corpus(1)
 dictionary = defaultdict(int)
 special_char = string.punctuation
for i in corpus:
if i in special_char:
dictionary[i] +=1 
<feature_engineering>
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )
corpus = create_corpus(0)
 dictionary = defaultdict(int)
 special_char = string.punctuation
for i in corpus:
if i in special_char:
dictionary[i] +=1<import_modules>
for i, name in tqdm(enumerate(submit['id'])) :
path = os.path.join('.. /input/imet-2019-fgvc6/test/', name)
 image = data_generator.load_image(path,(SIZE,SIZE,3))
score_predict = model.predict(image[np.newaxis]/255.)
label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )
<count_values><EOS> 
submit['attribute_ids'] = predicted
submit.to_csv('submission.csv', index=False )