Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<concatenate>
pd.set_option("max_columns",300)
 pd.set_option("max_rows",1103)
 tqdm.pandas()
df = pd.concat([train, test])
 df.shape<import_modules>
df_train = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_path = '.. /input/imet-2019-fgvc6/train/'
label_df = pd.read_csv('.. /input/imet-2019-fgvc6/labels.csv' )
import re<drop_column>
label_names = label_df['attribute_name'].values
num_labels = np.zeros(( df_train.shape[0],))
train_labels = np.zeros(( df_train.shape[0], len(label_names)))
 for row_index, row in enumerate(df_train['attribute_ids']):
num_labels[row_index] = len(row.split())
 for label in row.split() :
train_labels[row_index, int(label)] = 1
def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text )<feature_engineering>
culture, tag, unknown = 0, 0, 0
for l in label_names:
if l[:3] == 'cul':
culture += 1
elif l[:3] == 'tag':
tag += 1
else:
unknown += 1
print(f'Culture : {culture}')
 print(f'Tag : {tag}')
 print(f'Unknown : {unknown}')
 print(f'Total : {culture + tag + unknown}' )
df["text"] = df["text"].apply(lambda x: remove_url(x))<drop_column>
label_sum = np.sum(train_labels, axis=0)
 culture_sequence = label_sum[:398].argsort() [::-1]
tag_sequence = label_sum[398:].argsort() [::-1]
culture_labels = [label_names[x][9:] for x in culture_sequence]
culture_counts = [label_sum[x] for x in culture_sequence]
tag_labels = [label_names[x + 398][5:] for x in tag_sequence]
tag_counts = [label_sum[x + 398] for x in tag_sequence]
def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'', text )<feature_engineering>
df = pd.DataFrame({'Culture_label': culture_labels,'Culture_count': culture_counts})
 df.to_csv('cutr_labe.csv',index=True )
df["text"] = df["text"].apply(lambda x: remove_html(x))<drop_column>
for i in range(len(tag_labels)) :
print(tag_labels[i],':',tag_counts[i] )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 return emoji_pattern.sub(r'', text )<feature_engineering>
df = pd.DataFrame({'Tags_label': tag_labels,'Tags_count': tag_counts})
 df.to_csv('tags_label.csv',index=True )
df["text"] = df["text"].apply(lambda x: remove_emoji(x))<categorify>
train_attr_ohot = np.zeros(( len(df_train), len(label_df)) , dtype=int)
 for idx, attr_arr in enumerate(df_train.attribute_ids.str.split(" " ).apply(lambda l: list(map(int, l)) ).values):
train_attr_ohot[idx, attr_arr] = 1
def remove_punctuation(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table )<feature_engineering>
names_arr = label_df.attribute_name.values
df_train["attribute_names"] = [", ".join(names_arr[arr == 1])for arr in train_attr_ohot]
df["text"] = df["text"].apply(lambda x: remove_punctuation(x))<install_modules>
df_train["attr_num"] = train_attr_ohot.sum(axis=1)
 df_train["culture_attr_num"] = train_attr_ohot[:, :398].sum(axis=1)
 df_train["tag_attr_num"] = train_attr_ohot[:, 398:].sum(axis=1 )
!pip install pyspellchecker<import_modules>
i = 1
df_train["attribute_ids"]=df_train["attribute_ids"].apply(lambda x:x.split(" "))
df_train["id"]=df_train["id"].apply(lambda x:x+".png")
 plt.figure(figsize=[30,30])
 for img_name in os.listdir(".. /input/imet-2019-fgvc6/train/")[5:10]:
img = cv2.imread(".. /input/imet-2019-fgvc6/train/%s" % img_name)[...,[2, 1, 0]]
plt.subplot(5, 1, i)
 plt.imshow(img)
 ids = df_train[df_train["id"] == img_name]["attribute_ids"]
print(ids)
 title_val = []
for tag_id in ids.values[0]:
att_name = label_df[label_df['attribute_id'].astype(str)== tag_id]['attribute_name'].values[0]
title_val.append(att_name)
 plt.title(title_val)
 i += 1
plt.show()
from spellchecker import SpellChecker<string_transform>
import os
import numpy as np
from PIL import Image
import pandas as pd
import hashlib
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text )<import_modules>
def check_md5(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb")as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
 return hash_md5.hexdigest()
from tqdm import tqdm
from nltk.tokenize import word_tokenize<string_transform>
file_names = []
path_root = '.. /input/imet-2019-fgvc6/train/'
for filename in os.listdir(path_root)[0:20]:
file_names.append(check_md5(path_root+filename))
print(len(file_names))
def create_corpus(df):
corpus = []
for tweet in tqdm(df["text"]):
words = [word.lower() for word in word_tokenize(tweet)if \ 
(( word.isalpha() == 1)&(word not in stop)) ]
corpus.append(words)
 return corpus<statistical_test>
unit = np.unique(file_names,return_counts=True)
 count = 0
for i in range(len(unit[1])) :
if unit[1][i]>1:
count += 1
print('Duplicated Images')
 if count == 0:
print('NOT Duplicated Images' )
corpus = create_corpus(df )<feature_engineering>
def padding_image(path_img):
image_old = Image.open(path_img)
 width, height = image_old.size
if width > height:
distance_max = width
array = np.zeros([distance_max, distance_max, 3], dtype=np.uint8)
 array.fill(0)
 image_new = Image.fromarray(array)
 xmin = 0
ymin = int(( distance_max / 2)-(height / 2))
xmax = distance_max
ymax = int(( distance_max / 2)+(height / 2))
image_new.paste(image_old,(xmin, ymin, xmax, ymax))
return image_new
elif width < height:
distance_max = height
array = np.zeros([distance_max, distance_max, 3], dtype=np.uint8)
 array.fill(0)
 image_new = Image.fromarray(array)
 xmin = int(( distance_max / 2)-(width / 2))
ymin = 0
xmax = int(( distance_max / 2)+(width / 2))
ymax = distance_max
image_new.paste(image_old,(xmin, ymin, xmax, ymax))
return image_new
else:
return image_old
embedding_dict = {}
with open('.. /input/glove6b100dtxt/glove.6B.100d.txt','r')as glove:
for line in glove:
values = line.split()
word = values[0]
vectors = np.asarray(values[1:], 'float32')
 embedding_dict[word] = vectors
glove.close()<import_modules>
fastai.__version__
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Dense,SpatialDropout1D, Dropout, Input
from keras.initializers import Constant
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam
from keras import regularizers
from transformers import TFAutoModel, AutoTokenizer
from keras.models import Model<string_transform>
BATCH = 126
SIZE = 250
path = Path('.. /input/imet-2019-fgvc6/' )
MAX_LEN = 50
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences = tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad = pad_sequences(sequences,
maxlen = MAX_LEN,
truncating = 'post',
padding = 'post' )<count_unique_values>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet50/resnet50.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet50.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
word_index = tokenizer_obj.word_index
print('number of unique words: ', len(word_index))<feature_engineering>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
num_words = len(word_index)+ 1
embedding_matrix = np.zeros(( num_words,100))
for word, i in tqdm(word_index.items()):
if i > num_words:
continue
embedding_vector = embedding_dict.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<categorify>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
def encode_tweets(tokenizer, tweets, max_len):
nb_tweets = len(tweets)
 tokens = np.ones(( nb_tweets,max_len),dtype='int32')
 masks = np.zeros(( nb_tweets,max_len),dtype='int32')
 segs = np.zeros(( nb_tweets,max_len),dtype='int32')
 for k in range(nb_tweets):
tweet = tweets[k]
enc = tokenizer.encode(tweet)
 if len(enc)< max_len-2:
tokens[k,:len(enc)+2] = [0] + enc + [2]
masks[k,:len(enc)+2] = 1
else:
tokens[k,:max_len] = [0] + enc[:max_len-2] + [2]
masks[k,:max_len] = 1
return tokens,masks,segs<choose_model_class>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
def build_model(max_len):
ids = Input(( max_len,), dtype=tf.int32)
 attention = Input(( max_len,), dtype=tf.int32)
 token = Input(( max_len,), dtype=tf.int32)
 bertweet = TFAutoModel.from_pretrained("vinai/bertweet-base")
 x,_ = bertweet(ids,attention_mask=attention,token_type_ids=token)
 out = Dense(1,activation='sigmoid' )(x[:,0,:])
 model = Model(inputs=[ids, attention, token], outputs = out)
 optimizer = Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
model = build_model(MAX_LEN)
 model.summary()<choose_model_class>
tfms = get_transforms(do_flip=True, flip_vert=False, max_rotate=0.10, max_zoom=1.5, max_warp=0.2, max_lighting=0.2,
xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)) ,] )
optimizer = Adam(learning_rate=1e-5)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ["accuracy"] )<split>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
test_data = tweet_pad[train.shape[0]:]<load_pretrained>
learn = cnn_learner(data, base_arch=models.resnet50, loss_func=FocalLoss() , metrics=fbeta )
tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-base" )<categorify>
learn.load('stage-1' )
train_tokens, train_masks, train_segs = encode_tweets(tokenizer, train["text"].to_list() , MAX_LEN)
 train_labels = train["target"]<choose_model_class>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
ES = EarlyStopping(monitor='val_loss', mode='min', patience=10, restore_best_weights=True, verbose=1 )<train_model>
learn.lr_find()
learn.recorder.plot()
hist = model.fit([train_tokens,train_masks,train_segs], train_labels,
batch_size = 32, epochs = 50,
validation_split = 0.1, callbacks= [ES] )<load_from_csv>
learn.unfreeze()
learn.fit_one_cycle(16, slice(1e-4,1e-3))
learn.freeze()
learn.save('stage-2', return_path=True )
submit = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<categorify>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
test_tokens, test_masks, test_segs = encode_tweets(tokenizer,test["text"].to_list() , MAX_LEN )<save_to_csv>
test_preds = learn.TTA(ds_type=DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
test["target"] = model.predict([test_tokens, test_masks, test_segs] ).round().astype(int)
 submission = test[["id", "target"]]
submission.to_csv("submission.csv",index=False )<set_options>
test_df.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
learn.lr_find()
learn.recorder.plot()