Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<compute_train_metric>
tf.enable_eager_execution()
AUTOTUNE = tf.data.experimental.AUTOTUNE
print(os.listdir(".. /input"))
thresholds = np.linspace(0.2, 0.9, 31)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<define_variables>
SIZE = 156
BATCH_SIZE = 64
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<predict_on_test>
train_val_df = pd.read_csv(".. /input/imetmetadata/weird_images_w_labels.csv")
 train_val_df.head()
test_fn = glob.glob('./test/*')
 x_test = [cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in test_fn]
x_test = np.array(x_test)
 x_test = np.array([np.repeat(im[...,None],3,2)for im in x_test])
 print(x_test.shape)
 preds_test_orig = model.predict(x_test,batch_size=batch_size)
 x_test = np.array([np.fliplr(x)for x in x_test])
 preds_test_flipped = model.predict(x_test,batch_size=batch_size)
 preds_test_flipped = np.array([np.fliplr(x)for x in preds_test_flipped])
 preds_test = 0.5*preds_test_orig + 0.5*preds_test_flipped 
<categorify>
labels_df = pd.read_csv(".. /input/imet-2019-fgvc6/labels.csv")
 labels_df.head()
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 rles = []
i,max_img = 1,10
plt.figure(figsize=(16,4))
for p in tqdm_notebook(preds_test):
p = p.squeeze()
im = cv2.resize(p,(1024,1024))
im = im > threshold_best
im =(im.T*255 ).astype(np.uint8)
 rles.append(mask2rle(im, 1024, 1024))
i += 1
if i<max_img:
plt.subplot(1,max_img,i)
 plt.imshow(im)
 plt.axis('off' )<prepare_output>
N_CLASSES = len(labels_df)
 print("Number of classes:", N_CLASSES )
ids = [o.split('/')[-1][:-4] for o in test_fn]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.head()<save_to_csv>
n_width_300 = len(train_val_df[train_val_df['width'] == 300])
 n_height_300 = len(train_val_df[train_val_df['height'] == 300])
 n_width_height_300 = len(train_val_df[(train_val_df['height'] == 300)&(train_val_df['width'] == 300)])
 n_all_images = len(train_val_df)
 print("Total images: ", n_all_images)
 print("Total images with width or height = 300: ", n_height_300 + n_width_300 - n_width_height_300 )
sub_df.to_csv('submission.csv', index=False )<load_from_csv>
width_larger_equal_height = train_val_df[train_val_df['width'] >= train_val_df['height']]
width_smaller_equal_height = train_val_df[train_val_df['width'] < train_val_df['height']]
print("Number of image with width >= height: ", len(width_larger_equal_height))
print("Number of image with height = 300: ", sum(width_larger_equal_height['height'] == 300))
print("
")
 print("Number of image with width <= height: ", len(width_smaller_equal_height))
print("Number of image with width = 300: ", sum(width_smaller_equal_height['width'] == 300))
tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
<filter>
test_ids = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')['id']
print("Total number of images in test set: ", len(test_ids))
test_ids.head()
tweets[tweets.sent_len > 8]
tweets[tweets.word_len > 50]<string_transform>
train_val_df['aspect_ratio'] = train_val_df['width'] / train_val_df['height']
stop = set(stopwords.words('english'))
corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(tweet)]
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(tweet)]
def count_top_stopwords(corpus):
stopwords_freq = {}
for word in corpus:
if word in stop:
if word in stopwords_freq:
stopwords_freq[word] += 1
else:
stopwords_freq[word] = 1
topwords = sorted(stopwords_freq.items() , key=lambda item: item[1], reverse=True)[:10]
x,y = zip(*topwords)
 return x,y
x0,y0 = count_top_stopwords(corpus0)
 x1,y1 = count_top_stopwords(corpus1)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 plt.bar(x0,y0, color=color[0])
 plt.title('Top Stopwords for Disaster Tweets')
 plt.subplot(1,2,2)
 plt.bar(x1,y1, color=color[1])
 plt.title('Top Stopwords for Non-Disaster Tweets')
<string_transform>
ar_larger_2 = train_val_df[(train_val_df['aspect_ratio'] > 3)&(train_val_df['height'] == 300)]
ar_larger_2['id'].head()
stop = ENGLISH_STOP_WORDS.union(stop)
 def remove_url(txt):
return " ".join(re.sub("([^0-9A-Za-z \t])|(\w+:\/\/\S+)", "", txt ).split())
 corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(remove_url(tweet)) ]
corpus0 = list(filter(lambda x: x not in stop, corpus0))
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(remove_url(tweet)) ]
corpus1 = list(filter(lambda x: x not in stop, corpus1))
a = Counter(corpus0 ).most_common()
df0 = pd.DataFrame(a, columns=['Word','Count'])
 a = Counter(corpus1 ).most_common()
df1 = pd.DataFrame(a, columns=['Word','Count'])
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 sns.barplot(x='Word',y='Count',data=df0.head(10), color=color[1] ).set_title('Most Common Words for Non-Disasters')
 plt.xticks(rotation=45)
 plt.subplot(1,2,2)
 sns.barplot(x='Word',y='Count',data=df1.head(10), color=color[0] ).set_title('Most Common Words for Disasters')
 plt.xticks(rotation=45)
<feature_engineering>
ar_smaller_1_2 = train_val_df[(train_val_df['aspect_ratio'] < 1/3.0)&(train_val_df['width'] == 300)]
ar_smaller_1_2['id'].head()
tweets['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in tweets.text]
tweets['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in tweets.text]
tweets['exclaimation_num'] = [tweet.count('!')for tweet in tweets.text]
tweets['questionmark_num'] = [tweet.count('?')for tweet in tweets.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in tweets.text])
 tweets = tweets.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
tweets['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in tweets.text] 
<categorify>
categorical_labels = np.zeros(shape=(len(train_val_df['attribute_ids']), N_CLASSES))
for i, label in enumerate(train_val_df['attribute_ids']):
list_indices = [int(k)for k in label.split(" ")]
categorical_labels[i, list_indices] = 1
print(categorical_labels.shape )
tweets.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
tweets.text = [decontraction(tweet)for tweet in tweets.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
tweets.text = tweets.text.apply(lambda x: remove_emoji(x))<categorify>
def shift_img(img, width_shift_range, height_shift_range):

 img_shape = img.get_shape()
if width_shift_range or height_shift_range:
if width_shift_range:
width_shift_range = tf.random_uniform([],
-width_shift_range * img_shape[1].value,
width_shift_range * img_shape[1].value)
 if height_shift_range:
height_shift_range = tf.random_uniform([],
-height_shift_range * img_shape[0].value,
height_shift_range * img_shape[0].value)
 output_img = tf.contrib.image.translate(img,
[width_shift_range, height_shift_range])
 return output_img
else:
return img
def rotate_img(img, angle_range):
rotate_angle = tf.random_uniform([], minval = angle_range[0]/180.0*3.1415, maxval=angle_range[1]/180.0*3.1415)
 out_img = tf.contrib.image.rotate(img, angles=rotate_angle)
 return out_img
def flip_img(horizontal_flip, img):
if horizontal_flip:
flip_prob = tf.random_uniform([], 0.0, 1.0)
 out_img = tf.cond(tf.less(flip_prob, 0.5),
lambda: tf.image.flip_left_right(img),
lambda: img)
 return out_img
else:
return img
tweets.text = tweets.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 tweets.text = tweets.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
tweets.text = tweets.text.apply(lambda x: replace_amp(x)) 
<load_from_csv>
tr_cfg = {
'resize': [SIZE, SIZE],
'scale': 1 / 255.,
'horizontal_flip': True,
'width_shift_range': 0.2,
'height_shift_range': 0.2,
'rotation_range':(-15,15)
 }
tr_preprocessing_fn = functools.partial(_augment, **tr_cfg)
tweets = pd.read_csv('.. /input/twitter-logo/tweets_segmented.csv' )<string_transform>
val_cfg = {
'resize': [SIZE, SIZE],
'scale': 1 / 255.,
}
val_preprocessing_fn = functools.partial(_augment, **val_cfg)
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 tweets.text = tweets.text.apply(lambda x: lemma(x)) 
<string_transform>
def generate_kfolds(n_folds=10, image_names=None, categorical_labels=None):
return KFold(n_splits=n_folds, random_state=333, shuffle=True )
def generate_ngrams(text, n):
words = word_tokenize(text)
 return [' '.join(ngram)for ngram in list(get_data(ngrams(words, n)))if not all(w in stop for w in ngram)]
def get_data(gen):
try:
for elem in gen:
yield elem
except(RuntimeError, StopIteration):
return<categorify>
gamma = 2.0
epsilon = tf.keras.backend.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = tf.keras.backend.clip(pt, epsilon, 1-epsilon)
 CE = -tf.log(pt)
 FL = tf.pow(1-pt, gamma)* CE
loss = tf.keras.backend.sum(FL, axis=1)
 return loss
def f2_score(y_true, y_pred):
beta = 2
true_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_true * y_pred, 0, 1)) , axis=1)
 predicted_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_pred, 0, 1)) , axis=1)
 possible_positives = tf.keras.backend.sum(tf.round(tf.keras.backend.clip(y_true, 0, 1)) , axis=1)
 precision = true_positives /(predicted_positives + tf.keras.backend.epsilon())
 recall = true_positives /(possible_positives + tf.keras.backend.epsilon())
 return tf.keras.backend.mean(((1+beta**2)*precision*recall)/(( beta**2)*precision+recall+tf.keras.backend.epsilon()))
bigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_d_dict = {}
for bgs in bigrams_disaster:
for bg in bgs:
if bg in bigrams_d_dict:
bigrams_d_dict[bg] += 1
else:
bigrams_d_dict[bg] = 1
bigrams_d_df = pd.DataFrame(bigrams_d_dict.items() , columns=['Bigrams','Count'])
 bigrams_nd_dict = {}
for bgs in bigrams_ndisaster:
for bg in bgs:
if bg in bigrams_nd_dict:
bigrams_nd_dict[bg] += 1
else:
bigrams_nd_dict[bg] = 1
bigrams_nd_df = pd.DataFrame(bigrams_nd_dict.items() , columns=['Bigrams','Count'] )<categorify>
custom_weight = True
training = False
trigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_d_dict = {}
for tgs in trigrams_disaster:
for tg in tgs:
if tg in trigrams_d_dict:
trigrams_d_dict[tg] += 1
else:
trigrams_d_dict[tg] = 1
trigrams_d_df = pd.DataFrame(trigrams_d_dict.items() , columns=['Trigrams','Count'])
 trigrams_nd_dict = {}
for tgs in trigrams_ndisaster:
for tg in tgs:
if tg in trigrams_nd_dict:
trigrams_nd_dict[tg] += 1
else:
trigrams_nd_dict[tg] = 1
trigrams_nd_df = pd.DataFrame(trigrams_nd_dict.items() , columns=['Trigrams','Count'] )<feature_engineering>
def create_model(input_shape, n_out):
input_tensor = tf.keras.layers.Input(shape=input_shape)
 base_model = tf.keras.applications.Xception(include_top=False,
weights=None,
input_tensor=input_tensor)
 if(not custom_weight):
base_model.load_weights('.. /input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)
 x = tf.keras.layers.Dense(1024, activation='relu' )(x)
 final_output = tf.keras.layers.Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = tf.keras.Model(input_tensor, final_output)
 if(custom_weight):
model.load_weights('.. /input/imetpretrained/imet_xception_val_f2_0.686829-f2_0.611568.h5')
 return model
def remove_stopwords(text):
word_tokens = word_tokenize(text)
 return ' '.join([w.lower() for w in word_tokens if not w.lower() in stop])
 tweets['text_nostopwords'] = tweets.text.apply(lambda x: remove_stopwords(x)) 
<feature_engineering>
checkpoint = tf.keras.callbacks.ModelCheckpoint('.. /working/imet_xception_val_f2_{val_f2_score:3f}-f2_{f2_score:3f}.h5',
monitor='val_loss', verbose=1, save_best_only=True, mode='min',
save_weights_only = True)
 reduceLROnPlat = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = tf.keras.callbacks.EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = tf.keras.callbacks.CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
mask = np.array(Image.open('.. /input/twitter-logo/Twitter-Logo_white.png'))
reverse = mask[...,::-1,:]
def wc_words(target, mask=mask):
words = [word.lower() for tweet in tweets[tweets.target == target].text_nostopwords for word in tweet.split() ]
words = list(filter(lambda w: w != 'like', words))
words = list(filter(lambda w: w != 'new', words))
words = list(filter(lambda w: w != 'people', words))
dict = {}
for w in words:
if w in dict:
dict[w] += 1
else:
dict[w] = 1
my_cloud = WordCloud(background_color='white', stopwords=stop, mask=mask, random_state=0 ).generate_from_frequencies(dict)
 plt.subplot(1,2,target+1)
 plt.imshow(my_cloud, interpolation='bilinear')
 plt.axis("off")
 plt.figure(figsize=(15,10))
wc_words(0)
 plt.title('Non-Disaster')
 wc_words(1, reverse)
 plt.title('Disaster')
 plt.show()<drop_column>
if(training and not custom_weight):
n_folds = 10
for layer in model.layers:
layer.trainable = False
for i in range(-5,0):
model.layers[i].trainable = True
model.compile(
 loss=focal_loss,
optimizer=tf.keras.optimizers.Adam(1e-3),
metrics=[f2_score])
 kfolder = generate_kfolds(n_folds, image_names=train_val_df['id'], categorical_labels=categorical_labels)
 list_loss = []
list_f2 = []
for i,(train_indices, val_indices)in enumerate(kfolder.split(train_val_df['id'], categorical_labels)) :
print("============== Fold",(i+1), "================")
 train_dataset = create_dataset(train_val_df.loc[train_indices, 'id'], categorical_labels[train_indices, :], preproc_fn=tr_preprocessing_fn, threads=2)
 val_dataset = create_dataset(train_val_df.loc[val_indices, 'id'], categorical_labels[val_indices, :], preproc_fn=val_preprocessing_fn, threads=2)
 history = model.fit_generator(
 train_dataset,
steps_per_epoch=int(np.ceil(float(len(train_indices)) / float(BATCH_SIZE))),
validation_data=val_dataset,
validation_steps=int(np.ceil(float(len(val_indices)) / float(BATCH_SIZE))),
epochs=1,
verbose=1,
callbacks=callbacks_list)
 list_loss.append(history.history['val_loss'][0])
 list_f2.append(history.history['val_f2_score'][0])
 print("Val loss of fold %d = %f"%(i+1, history.history['val_loss'][0]))
print("F2 score of fold %d = %f"%(i+1, history.history['val_f2_score'][0]))
print("
Mean val loss over all: ", np.mean(np.array(list_loss)))
 print("Mean f2_score over all: ", np.mean(np.array(list_f2)) )
pd.reset_option('max_colwidth')
 tweets.drop('text_nostopwords', axis=1, inplace=True)
 tweets.head()<split>
if(training):
n_folds = 15
for layer in model.layers:
layer.trainable = True
model.compile(loss=focal_loss,
optimizer=tf.keras.optimizers.Adam(lr=0.0003),
metrics=[f2_score])
 checkpoint = tf.keras.callbacks.ModelCheckpoint('.. /working/imet_xception_val_f2_{val_f2_score:3f}-f2_{f2_score:3f}.h5',
monitor='val_loss', verbose=1, save_best_only=True,
mode='min', save_weights_only = True)
 reduceLROnPlat = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1,
verbose=1, mode='auto', epsilon=0.0001)
 early = tf.keras.callbacks.EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
print("Start training...")
 kfolder = generate_kfolds(n_folds, image_names=train_val_df['id'], categorical_labels=categorical_labels)
 list_loss = []
list_f2 = []
for i,(train_indices, val_indices)in enumerate(kfolder.split(train_val_df['id'], categorical_labels)) :
print("============== Fold",(i+1), "================")
 train_dataset = create_dataset(train_val_df.loc[train_indices, 'id'], categorical_labels[train_indices, :], preproc_fn=tr_preprocessing_fn, threads=2)
 val_dataset = create_dataset(train_val_df.loc[val_indices, 'id'], categorical_labels[val_indices, :], preproc_fn=val_preprocessing_fn, threads=2)
 history = model.fit_generator(
 train_dataset,
steps_per_epoch=int(np.ceil(float(len(train_indices)) / float(BATCH_SIZE))),
validation_data=val_dataset,
validation_steps=int(np.ceil(float(len(val_indices)) / float(BATCH_SIZE))),
epochs=1,
verbose=1,
callbacks=callbacks_list)
 list_loss.append(history.history['val_loss'][0])
 list_f2.append(history.history['val_f2_score'][0])
 print("Val loss of fold %d = %f"%(i+1, history.history['val_loss'][0]))
print("F2 score of fold %d = %f"%(i+1, history.history['val_f2_score'][0]))
print("
Mean val loss over all: ", np.mean(np.array(list_loss)))
 print("Mean f2_score over all: ", np.mean(np.array(list_f2)) )
X_train, X_val, y_train, y_val = train_test_split(tweets.drop(['id','keyword','location','target'],axis=1), tweets[['target']], test_size=0.2, stratify=tweets[['target']], random_state=0)
 X_train_text = X_train['text']
X_val_text = X_val['text']
print('X_train shape: ', X_train.shape)
 print('X_val shape: ', X_val.shape)
 print('y_train shape: ', y_train.shape)
 print('y_val shape: ', y_val.shape )<count_values>
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 predicted = []
print('Train Class Proportion:
', y_train['target'].value_counts() / len(y_train)* 100)
 print('
Validation Class Proportion:
', y_val['target'].value_counts() / len(y_val)* 100 )<train_model>
def file_filter(filename):
return filename.startswith('imet_xception')
 if(training):
model_files = list(filter(file_filter, os.listdir('.. /working/')))
 model_files = sorted(model_files)
 weight_path = os.path.join(".. /working/", model_files[-1])
 print("Current weight path for prediction: ", weight_path)
 model.load_weights(weight_path )
tokenizer_1 = Tokenizer(num_words=5000, oov_token='<UNK>')
 tokenizer_1.fit_on_texts(X_train_text )<string_transform>

 BATCH = 512
check_dataset = create_dataset(train_val_df['id'], categorical_labels, preproc_fn=val_preprocessing_fn, threads=2)
 counter = 0
n_val = round(len(train_val_df)*0.5//BATCH)
 print(n_val)
 lastFullValPred = np.empty(( 0, N_CLASSES))
lastFullValLabels = np.empty(( 0, N_CLASSES))
for X, y in tqdm(check_dataset):
if(counter >= n_val):
break
else:
counter += 1
y_pred = model.predict(X.numpy())
 lastFullValPred = np.append(lastFullValPred, y_pred, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, y, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )
X_train_text = tokenizer_1.texts_to_sequences(X_train_text)
 X_val_text = tokenizer_1.texts_to_sequences(X_val_text)
 print(X_train_text[:10])
 print('')
 print(X_val_text[:10] )<data_type_conversions>
beta_f2 = 2
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return f2
def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score
tokenizer_1.sequences_to_texts([X_train_text[1]] )<feature_engineering>
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )
vocab_size = len(tokenizer_1.word_index)+ 1
embeddings_index = dict()
f = open('.. /input/glove-global-vectors-for-word-representation/glove.twitter.27B.200d.txt')
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))<categorify>
def load_img(img_name):
path = '.. /input/imet-2019-fgvc6/test/'+ img_name +".png"
img_string = tf.read_file(path)
 img = tf.image.decode_png(img_string)
 shape = tf.shape(img)
 tfwidth = tf.to_float(shape[1])
 tfheight = tf.to_float(shape[0])
 img = tf.cond(tf.math.logical_or(tfwidth/tfheight > 3.0, tfwidth/tfheight < 1/3.0),
lambda: tf.image.random_crop(img, size=[300, 300, 3]),
lambda: img)
 img = tf.image.resize(img,(SIZE, SIZE))
img = tf.to_float(img)/ 255.0
return img
embedding_matrix = np.zeros(( vocab_size, 200))
for word, i in tokenizer_1.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Embedding Matrix Shape:', embedding_matrix.shape )<count_values>
submit_ds = tf.data.Dataset.from_tensor_slices(submit['id'])
 submit_ds = submit_ds.apply(tf.data.experimental.map_and_batch(map_func=load_img, batch_size=BATCH_SIZE))
num_epochs=15
dropout=0.2
recurrent_dropout=0.2
lr=0.0005
batch_size=128
class_weight = {0: y_train['target'].value_counts() [1]/len(y_train), 1: y_train['target'].value_counts() [0]/len(y_train)}<choose_model_class>
predictions = model.predict_generator(submit_ds, steps = int(np.ceil(len(submit['id'])/ BATCH_SIZE)) , verbose=1 )
lstm_model = Sequential()
embedding_layer = Embedding(vocab_size, 200, weights=[embedding_matrix], input_length=maxlen, trainable=False)
 lstm_model.add(embedding_layer)
 lstm_model.add(LSTM(128, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
lstm_model.add(LSTM(128))
lstm_model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=lr)
 lstm_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(lstm_model.summary())
<train_model>
label_indices = []
for raw_pred in predictions:
label_predict = np.arange(N_CLASSES)[raw_pred>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 label_indices.append(str_predict_label )
checkpoint = ModelCheckpoint('lstm_model.h5', monitor='val_acc', save_best_only=True)
 history = lstm_model.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history )<choose_model_class>
submit['attribute_ids'] = label_indices
submit.to_csv('submission.csv', index=False )
<normalization>
submit['attribute_ids'].map(lambda row: len(row.split(" ")) ).value_counts()