Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<import_modules>
fastai.__version__
nlp = spacy.load("en")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for dirname, _, filenames in os.walk('/kaggle/working'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
BATCH = 64
SIZE = 250
path = Path('.. /input/imet-2019-fgvc6/' )
EMBEDDING_SOURCE = "GLOVE"
TUNER_ON = True
DASK_ON = False<load_from_csv>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet152/resnet152.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet152.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 train_df.shape<count_values>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
train_df.keyword.value_counts()<count_values>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
train_df.location.value_counts()<categorify>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
def keyword_cleaner(x):
if type(x)is float:
return 'UNKNOWN'
else:
x = x.replace('%20', ' ')
 doc = nlp(x)
 full_lemma = ' '.join(y.lemma_ for y in doc)
 return full_lemma
train_df['keyword_cleaned'] = train_df['keyword'].map(keyword_cleaner)
 keyword_enc = OneHotEncoder(handle_unknown='ignore')
 transformed = keyword_enc.fit_transform(train_df['keyword_cleaned'].to_numpy().reshape(-1, 1))
ohe_df = pd.DataFrame(transformed.todense() , columns=keyword_enc.get_feature_names())
 meta_features = len(ohe_df.columns)
 ohe_df.to_csv('ohe_train_df.csv', index=False)
 dump(keyword_enc, 'keyword_encoder.joblib')
<categorify>
tfms = get_transforms(do_flip=True, flip_vert=False, max_rotate=0.10, max_zoom=1.5, max_warp=0.2, max_lighting=0.2,
xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)) ,] )
def location_cleaner(x):
if type(x)is float:
return 'unknown'
else:
x = x.lower().strip()
x = x.replace('?', '')
 if len(x)== 0:
return 'unknown'
x = x.replace('united states', 'usa')
 x = x.replace('usa of america', 'usa')
 x = x.replace('norf carolina', 'north carolina')
 if x == 'tn':
x = 'tennessee'
return x
train_df['location_cleaned'] = train_df['location'].map(location_cleaner)
 location_enc = OneHotEncoder(handle_unknown='ignore')
 transformed = location_enc.fit_transform(train_df['location_cleaned'].to_numpy().reshape(-1, 1))
ohe_location_df = pd.DataFrame(transformed.todense() , columns=location_enc.get_feature_names())
 meta_location_features = len(ohe_location_df.columns)
 ohe_location_df.to_csv('ohe_location_train_df.csv', index=False)
 dump(location_enc, 'location_encoder.joblib' )<prepare_x_and_y>
data.show_batch()
train_x = train_df[['text']]
train_y = train_df[['target']]
input_shape = train_x.shape[0]
vocab = set()
for desc in train_df['text'].values:
vocab = vocab.union(set(desc.split(' ')))
 vocab_size = len(vocab)+ 1
print(vocab_size)
 print(train_x.shape)
 word_index_cnt = 1
word_index = {}
seq_of_seqs = []
rows = train_x['text'].tolist()
print(len(rows))
iters = 0
for snippet in train_x['text'].tolist() :
if iters % 1000 == 0:
print(iters)
 doc = nlp(snippet)
 sequences = []
for token in doc:
lower_token = token.text.lower()
if(lower_token not in word_index):
word_index[lower_token] = word_index_cnt
word_index_cnt += 1
sequences.append(word_index[lower_token])
 seq_of_seqs.append(sequences)
 iters += 1
word_index['<OOV>'] = word_index_cnt
word_index_cnt += 1
max_len = 25
train_x_new = pad_sequences(seq_of_seqs, maxlen=max_len)
 print('Done tokenization' )<load_pretrained>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
if EMBEDDING_SOURCE == 'FASTTEXT':
print('loading word embeddings...')
 embeddings_index = {}
with codecs.open('/kaggle/input/fasttext/wiki.simple.vec', encoding='utf-8')as fs:
for line in tqdm(fs):
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('preparing embedding matrix...')
 MAX_NB_WORDS = 100000
FASTTEXT_EMBEDDING_DIM = 300
words_not_found = []
nb_words = min(MAX_NB_WORDS, len(word_index)+ 1)
 embedding_matrix = np.zeros(( nb_words, FASTTEXT_EMBEDDING_DIM))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if(embedding_vector is not None)and len(embedding_vector)> 0:
embedding_matrix[i] = embedding_vector
else:
words_not_found.append(word)
 print('number of null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1)== 0))
fasttext_embedding_layer = layers.Embedding(nb_words, FASTTEXT_EMBEDDING_DIM, weights=[embedding_matrix], input_length=input_shape, trainable=False )<feature_engineering>
learn = cnn_learner(data, base_arch=models.resnet152, loss_func=FocalLoss() , metrics=fbeta )
if EMBEDDING_SOURCE == 'GLOVE':
print('Creating Glove Embedding Layer')
 embeddings_index = {}
GLOVE_DIR = '/kaggle/input/glove-global-vectors-for-word-representation'
GLOVE_EMBEDDING_DIM = 200
with open(os.path.join(GLOVE_DIR, 'glove.6B.200d.txt')) as fs:
lines = fs.readlines()
for line in lines:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
hits = 0
misses = 0
missed_words = []
embedding_matrix = np.zeros(( len(word_index)+ 1, GLOVE_EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
hits += 1
else:
misses += 1
missed_words.append(word)
 print("Converted %d words(%d misses)" %(hits, misses))
print(missed_words[0:100])
 glove_embedding_layer = layers.Embedding(
 len(word_index)+ 1,
GLOVE_EMBEDDING_DIM,
embeddings_initializer=initializers.Constant(embedding_matrix),
input_length=input_shape,
trainable=False,
)<load_pretrained>
learn.lr_find()
learn.recorder.plot()
if EMBEDDING_SOURCE == 'BERT':
print('Creating BERT Embedding Layer')
 BERT_EMBEDDING_DIM = 256
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 iters = 0
ii_list = []
am_list = []
tti_list = []
for snippet in train_x['text'].tolist() :
if iters % 1000 == 0:
print(iters)
 encoding = tokenizer(snippet)
 input_ids = encoding['input_ids']
token_type_ids = [0] * len(encoding['input_ids'])
 attention_mask = [1] * len(input_ids)
 padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 ii_list.append(input_ids)
 am_list.append(attention_mask)
 tti_list.append(token_type_ids)
 iters += 1
encoder = TFBertModel.from_pretrained("bert-base-uncased")
 print(encoder)
 input_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 token_type_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = layers.Input(shape=(max_len,), dtype=tf.int32)
 sequence_output, pooled_output = encoder({
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": attention_mask
})
 print(type(seq_output))
print(type(pooled_output))
print(type(encoder.bert))
bert_embedding_layer = encoder.bert
print('Done creating BERT Embedding layer' )<choose_model_class>
learn.unfreeze()
learn.fit_one_cycle(18, slice(1e-5,1e-2))
learn.freeze()
learn.save('stage-1', return_path=True )
if EMBEDDING_SOURCE == 'CUSTOM':
CUSTOM_EMBEDDING_DIM = 512
training_data_embedding_layer = layers.Embedding(input_dim = vocab_size, output_dim = CUSTOM_EMBEDDING_DIM, input_length = input_shape )<choose_model_class>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
def build_model(hp):
nlp_input = layers.Input(shape=(input_shape,), name='nlp_input')
 meta_input = layers.Input(shape=(meta_features + meta_location_features,), name='meta_input')
 if EMBEDDING_SOURCE == 'GLOVE':
print('GLOVE EMBARKED')
 FINAL_EMBEDDING_DIM = GLOVE_EMBEDDING_DIM
embedding_layer = glove_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'BERT':
print('BERT EMBARKED')
 FINAL_EMBEDDING_DIM = BERT_EMBEDDING_DIM
embedding_layer = bert_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'FASTTEXT':
print('FASTTEXT EMBARKED')
 FINAL_EMBEDDING_DIM = FASTTEXT_EMBEDDING_DIM
embedding_layer = fasttext_embedding_layer(nlp_input)
 else:
FINAL_EMBEDDING_DIM = CUSTOM_EMBEDDING_DIM
embedding_layer = training_data_embedding_layer(nlp_input)
 lstm_layer = layers.LSTM(FINAL_EMBEDDING_DIM, dropout=0.1 )(embedding_layer)
 merged = layers.Concatenate(axis=1 )([lstm_layer, meta_input])
 hp_dense_units = hp.Int('units', min_value = 32, max_value = 512, step = 32)
 dense_layer = layers.Dense(hp_dense_units, activation='relu' )(merged)
 output_layer = layers.Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=[nlp_input, meta_input], outputs=[output_layer])
 hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3, 1e-4])
 model.compile(loss='binary_crossentropy', optimizer = Adam(learning_rate=hp_learning_rate), metrics=['accuracy'],)
 return model<choose_model_class>
test_preds = learn.TTA(ds_type=DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
def build_model_wrapper(optimizer="adam", embedding_size = 512, dense_size=32, dropout_value = 0.1):
nlp_input = layers.Input(shape=(input_shape,), name='nlp_input')
 meta_input = layers.Input(shape=(meta_features + meta_location_features,), name='meta_input')
 if EMBEDDING_SOURCE == 'GLOVE':
print('GLOVE EMBARKED')
 FINAL_EMBEDDING_DIM = GLOVE_EMBEDDING_DIM
embedding_layer = glove_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'BERT':
print('BERT EMBARKED')
 FINAL_EMBEDDING_DIM = BERT_EMBEDDING_DIM
embedding_layer = bert_embedding_layer(nlp_input)
 elif EMBEDDING_SOURCE == 'FASTTEXT':
print('FASTTEXT EMBARKED')
 FINAL_EMBEDDING_DIM = FASTTEXT_EMBEDDING_DIM
embedding_layer = fasttext_embedding_layer(nlp_input)
 else:
FINAL_EMBEDDING_DIM = CUSTOM_EMBEDDING_DIM
embedding_layer = training_data_embedding_layer(nlp_input)
 lstm_layer = layers.LSTM(FINAL_EMBEDDING_DIM, dropout=dropout_value )(embedding_layer)
 merged = layers.Concatenate(axis=1 )([lstm_layer, meta_input])
 dense_layer = layers.Dense(dense_size, activation='relu' )(merged)
 output_layer = layers.Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=[nlp_input, meta_input], outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer = optimizer, metrics=['accuracy'],)
 print(model.summary())
 return model<choose_model_class>
test_df.to_csv('submission.csv', index=False )