Kaggle competition: iMet Collection 2019 - FGVC6
class attention(Layer):
def __init__(self,**kwargs):
super(attention,self ).__init__(**kwargs)
 def build(self,input_shape):
self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="normal")
 self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")
 super(attention, self ).build(input_shape)
 def call(self,x):
et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
 at=K.softmax(et)
 at=K.expand_dims(at,axis=-1)
 output=x*at
return K.sum(output,axis=1)
 def compute_output_shape(self,input_shape):
return(input_shape[0],input_shape[-1])
 def get_config(self):
return super(attention,self ).get_config()<choose_model_class>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 beta_f2=2
num_epochs=15
dropout=0.3
recurrent_dropout=0.3
lr=0.0005
batch_size=128
sequence_input = Input(shape=(maxlen,))
embedded_sequences = Embedding(vocab_size, 200, weights=[embedding_matrix], trainable=False )(sequence_input)
 lstm = Bidirectional(LSTM(128, return_sequences = True, dropout=dropout, recurrent_dropout=recurrent_dropout))(embedded_sequences)
(lstm, forward_h, forward_c, backward_h, backward_c)= Bidirectional(LSTM(128, return_sequences=True, return_state=True))(lstm)
 att_out=attention()(lstm)
 outputs=Dense(1,activation='sigmoid' )(att_out)
 model_attn = Model(sequence_input, outputs)
 adam = optimizers.Adam(lr=lr)
 model_attn.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(model_attn.summary() )<train_model>
nb_classes = 1103
img_size = 224
nb_epochs = 25
checkpoint = ModelCheckpoint('attn_model.h5', monitor='val_acc', save_best_only=True)
 history_attn = model_attn.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history_attn )<init_hyperparams>
imet_path = '.. /input/imet-2019-fgvc6/'
model_weights_path = '.. /input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5'
train_df_path = imet_path + 'train.csv'
label_df_path = imet_path + 'labels.csv'
sub_df_path = imet_path + 'sample_submission.csv'
train_img_path = imet_path + 'train/'
test_img_path = imet_path + 'test/'
maxlen = 160
lr = 1e-5
num_epochs = 5
batch_size=16<string_transform>
class AdamAccumulate_v1(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):
super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, dtype='int64')
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update(self.iterations, self.iterations + 1)]
flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)
 flag = K.cast(flag, K.floatx())
 self.updates.append(K.update(self.effective_iterations,
self.effective_iterations + K.cast(flag, 'int64')))
 lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.effective_iterations,
K.dtype(self.decay))))
t = K.cast(self.effective_iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):
gg_t =(1 - flag)*(gg + g)
 m_t =(self.beta_1 * m)+(1.- self.beta_1)*(gg + flag * g)/ K.cast(self.accum_iters, K.floatx())
 v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(
(gg + flag * g)/ K.cast(self.accum_iters, K.floatx()))
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - flag * lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
p_t = p - flag * lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(( m, flag * m_t +(1 - flag)* m))
self.updates.append(( v, flag * v_t +(1 - flag)* v))
self.updates.append(( gg, gg_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
path_to_train = imet_path+'train/'
data = pd.read_csv(imet_path+'train.csv')
 train_dataset_info = []
for name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info )
def build_model(bert_layer, max_len=512, lr=1e-5):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=lr), loss='binary_crossentropy', metrics=['accuracy'])
 return model<feature_engineering>
gamma = 2.0
epsilon = K.epsilon()
def old_focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = K.clip(pt, epsilon, 1-epsilon)
 CE = -K.log(pt)
 FL = K.pow(1-pt, gamma)* CE
loss = K.sum(FL, axis=1)
 return loss
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
train_df = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_labels = np.zeros(( train_df.shape[0], 1103))
for row_index, row in enumerate(train_df['attribute_ids']):
for label in row.split() :
train_labels[row_index, int(label)] = 1
labels_sum = np.sum(train_labels,axis=0 )
train_input = bert_encode(X_train.text.values, tokenizer, max_len=maxlen)
 val_input = bert_encode(X_val.text.values, tokenizer, max_len=maxlen)
 train_labels = y_train.target.values
val_labels = y_val.target.values<choose_model_class>
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 train_index = None
valid_index = None
class data_generator(Sequence):
def mix_up(x, y):
x = np.array(x, np.float32)
 lam = np.random.beta(0.4, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def create_train(dataset_info, batch_size, shape, augument=True, mix=False):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), nb_classes))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(( image)/255.0)
 batch_labels[i][X_train_batch[i]['labels']] = 1
if(mix):
batch_images, batch_labels = data_generator.mix_up(batch_images, batch_labels)
 yield np.array(batch_images, np.float32), batch_labels
def create_valid(valid_dataset_info, batch_size, shape, augument=False):
global valid_index
assert shape[2] == 3
while True:
for start in range(0, len(valid_dataset_info), batch_size):
end = min(start + batch_size, len(valid_dataset_info))
batch_images = []
X_train_batch = valid_dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), nb_classes))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(( image)/255.0)
 batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, shape):
img_cv = cv2.imread(path+'.png')
 img_cv = cv2.cvtColor(img_cv,cv2.COLOR_BGR2RGB)
 if img_cv.shape[1]==300 and img_cv.shape[0]>=450:
over_high = img_cv.shape[0]-300
start = np.random.randint(over_high//3,over_high,1)[0]
img_cv = img_cv[start:start+300,:,:]
elif img_cv.shape[0]==300 and img_cv.shape[1]>450:
over_width = img_cv.shape[1]-300
start = np.random.randint(over_width//3,over_width,1)[0]
img_cv = img_cv[:,start:start+300,:]
img_cv = cv2.resize(img_cv,(img_size,img_size))
return img_cv
def augment(image):
augment_img = iaa.Sequential([
iaa.SomeOf(( 0,4),[
iaa.ContrastNormalization(( 0.5, 1.5)) ,
iaa.Multiply(( 0.9, 1.1), per_channel=0.2),
iaa.Fliplr(0.5),
iaa.GaussianBlur(sigma=(0, 0.6)) ,
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug
bert_model = build_model(bert_layer, max_len=maxlen, lr=lr)
 bert_model.summary()<train_model>
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,Concatenate)
checkpoint = ModelCheckpoint('bertmodel.h5', monitor='val_accuracy', save_best_only=True)
 bert_history = bert_model.fit(
 train_input, train_labels,
validation_data=(val_input, val_labels),
epochs=num_epochs,
callbacks=[checkpoint],
batch_size=batch_size
)<train_model>
class CyclicLR(Callback):

 def __init__(self, base_lr=1e-5, max_lr=1e-2, step_size=1000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
clf = RandomForestClassifier(n_estimators=500, max_depth=15, min_samples_split=20, min_samples_leaf=2, n_jobs=-1, random_state=0)
 clf.fit(X_train.drop('text',axis=1), y_train.target.values)
 clf_pred = clf.predict_proba(X_val.drop('text',axis=1))
print('Validation Accuracy:', accuracy_score(y_val.target.values, clf_pred.argmax(axis=-1)) )<predict_on_test>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 model = applications.DenseNet121(weights=None,include_top=False,input_tensor=input_tensor)
 model.load_weights(model_weights_path)
 x0 = model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu', kernel_regularizer=l2(5e-4))(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
bert_model.load_weights('bertmodel.h5')
 val = X_val.copy()
val = val[['text']]
val['target'] = y_val
val['pred'] = bert_model.predict(val_input)
 val['pred'] = val['pred'].apply(lambda x: 1 if x >=0.5 else 0)
 error = val[val['target'] != val['pred']]
error.head()<feature_engineering>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 17; warmup_batch_size = 128;mixup_batch_size = 64
batch_size = 64
checkpoint = ModelCheckpoint('denseNet121_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=7)
 csv_logger = CSVLogger(filename='training_log.csv',
separator=',',
append=True)
 indexes = np.arange(train_dataset_info.shape[0])
 train_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)
 train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(img_size,img_size,3), augument=True, mix=False)
 train_mixup = data_generator.create_train(
 train_dataset_info[train_indexes], mixup_batch_size,(img_size,img_size,3), augument=False, mix=True)
 train_generator_warmup = data_generator.create_train(
 train_dataset_info[train_indexes], warmup_batch_size,(img_size,img_size,3), augument=False)
 validation_generator = data_generator.create_valid(
 train_dataset_info[valid_indexes], batch_size,(img_size,img_size,3), augument=False)
 lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),
epochs=1, batch_size=warmup_batch_size, verbose=0)
test['char_len'] = test.text.str.len()
word_tokens = [len(word_tokenize(tweet)) for tweet in test.text]
test['word_len'] = word_tokens
sent_tokens = [len(sent_tokenize(tweet)) for tweet in test.text]
test['sent_len'] = sent_tokens<feature_engineering>
model = create_model(
 input_shape=(img_size,img_size,3),
n_out=nb_classes )
test['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in test.text]
test['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in test.text]
test['exclaimation_num'] = [tweet.count('!')for tweet in test.text]
test['questionmark_num'] = [tweet.count('?')for tweet in test.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in test.text])
 test = test.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
test['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in test.text]<categorify>
%%time
for layer in model.layers:
layer.trainable = False
for i in range(-6,0):
model.layers[i].trainable = True
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(1e-3))
model.fit_generator(
 train_generator_warmup,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(warmup_batch_size)) ,
epochs=2,
max_queue_size=16,
workers=2,
use_multiprocessing=True,
verbose=1,
class_weight = 'auto'
)
test.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
test.text = [decontraction(tweet)for tweet in test.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
test.text = test.text.apply(lambda x: remove_emoji(x))<categorify>
%%time
for layer in model.layers:
layer.trainable = True
model.compile(
 loss='binary_crossentropy',
optimizer=AdamAccumulate(lr=1e-4, accum_iters=4))
model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(mixup_batch_size)) ,
epochs=7,
verbose=1,
class_weight = 'auto',
max_queue_size=16,
workers=WORKERS,
use_multiprocessing=True)
test.text = test.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 test.text = test.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
test.text = test.text.apply(lambda x: replace_amp(x)) 
<load_from_csv>
%%time
clr = CyclicLR(base_lr=1e-5,max_lr=6e-5,mode='triangular2', step_size=np.ceil(len(train_indexes)/batch_size))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=3e-5))
callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
class_weight = 'auto',
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list )
test = pd.read_csv('.. /input/twitter-logo/tweets_test_segmented.csv' )<string_transform>
print(os.listdir('.. /working/'))
model.save('.. /input/denseNet121.h5' )
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 test.text = test.text.apply(lambda x: lemma(x))<categorify>
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 predicted = []
test_text = test['text']
test_text = tokenizer_1.texts_to_sequences(test_text)
 test_text = pad_sequences(test_text, padding='post', maxlen=50)
 print('X_test shape:', test_text.shape )<save_to_csv>
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1), axis=1)
 tn = np.sum(( y_true == 0)&(y_pred == 0), axis=1)
 fp = np.sum(( y_true == 0)&(y_pred == 1), axis=1)
 fn = np.sum(( y_true == 1)&(y_pred == 0), axis=1)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return np.mean(f2)
 def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score
lstm_model.load_weights('lstm_model.h5')
 submission = test.copy() [['id']]
submission['target'] = lstm_model.predict_classes(test_text)
 submission.to_csv('submission.csv', index=False)
 display(submission.head())
<save_to_csv>
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )
model_attn.load_weights('attn_model.h5')
 submission_attn = test.copy() [['id']]
submission_attn['target'] = model_attn.predict(test_text)
 submission_attn['target'] = submission_attn['target'].apply(lambda x: 1 if x >=0.5 else 0)
 submission_attn.to_csv('submission_attn.csv', index=False)
 display(submission_attn.head() )<save_to_csv>
for i, name in tqdm(enumerate(submit['id'])) :
path = os.path.join('.. /input/imet-2019-fgvc6/test/', name)
 image = data_generator.load_image(path,(img_size,img_size,3))
score_predict = model.predict(( image[np.newaxis])/255.0)
 label_predict = np.arange(nb_classes)[score_predict[0]>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )
<save_to_csv><EOS> 
submit['attribute_ids'] = predicted
submit.to_csv('submission.csv', index=False )