Kaggle competition: Recursion Cellular Image Classification
max_len = 72
train_X = np.column_stack(( pad_sequences(train_tokenized, maxlen=max_len), train_meta))
test_X = np.column_stack(( pad_sequences(test_tokenized, maxlen=max_len), test_meta))
train_y = train_df.target.values<load_pretrained>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 224
NUM_CLASSES = 1108
news_path = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings = KeyedVectors.load_word2vec_format(news_path, binary=True )<define_variables>
df_train = pd.read_csv('.. /input/recursion-complete/new_train.csv')
 df_test = pd.read_csv('.. /input/recursion-complete/new_test.csv')
 x = df_train['filename']
y = df_train['sirna']
x, y = shuffle(x, y, random_state=10)
 y.hist()
mean = np.mean(embeddings.vectors)
 std = np.std(embeddings.vectors)
 _, size = embeddings.vectors.shape
num_words = min(len(tokenizer.word_index), max_features)
 embedding_matrix = np.random.normal(mean, std,(num_words, size))<feature_engineering>
y = to_categorical(y, num_classes=NUM_CLASSES )
for word, index in tokenizer.word_index.items() :
if index >= max_features: continue
try:
word_vector = embeddings.get_vector(word)
 embedding_matrix[index] = word_vector
except:
pass
print(embedding_matrix.shape )<set_options>
train_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,
stratify=y, random_state=10)
 print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape )
tqdm.pandas()
def seed_torch(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed = 0
seed_torch(seed)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))
class BiLSTM(nn.Module):
def __init__(self, embedding_matrix = embedding_matrix, embed_size = 300, hidden_size = 128):
super().__init__()
self.embedding = nn.Embedding(max_features, size)
 if embedding_matrix is not None:
self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional = True, batch_first = True, num_layers = 2, dropout = 0.1)
 self.fc = nn.Linear(hidden_size * 2 + 6, 1)
 def forward(self, x):
embedded = self.embedding(x[:,:72])
 f = torch.tensor(x[:,72:], dtype=torch.float32 ).cuda()
h_lstm,(h,c)= self.lstm(embedded)
 h_fc = torch.cat(( h[-1],h[-2],f), dim = 1)
 return self.fc(h_fc )<define_variables>
class My_Generator(Sequence):
def __init__(self, image_filenames, labels,
batch_size, is_train=True,
mix=False, augment=False):
self.image_filenames, self.labels = image_filenames, labels
self.batch_size = batch_size
self.is_train = is_train
self.is_augment = augment
if(self.is_train):
self.on_epoch_end()
self.is_mix = mix
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
return self.train_generate(batch_x, batch_y)
 return self.valid_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_train):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def mix_up(self, x, y):
lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def train_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/recursion-complete/train/train/'+sample)
 if(self.is_augment):
img = seq.augment_image(img)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/255
batch_y = np.array(batch_y, np.float32)
 if(self.is_mix):
batch_images, batch_y = self.mix_up(batch_images, batch_y)
 return batch_images, batch_y
def valid_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/recursion-complete/train/train/'+sample)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=0 ).split(train_X, train_y))
train_epochs = 5
batch_size = 64
print(type(train_X), train_X.shape)
 print(type(train_y), train_y.shape)
 train_X = torch.Tensor(train_X)
 train_y = torch.Tensor(train_y)
 train_meta = torch.Tensor(train_meta)
 test_meta = torch.Tensor(test_meta)
 train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
print('Fold:{0}'.format(i))
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train_data = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid_data = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, shuffle=False)
 model = BiLSTM()
model.cuda()
loss_function = nn.BCEWithLogitsLoss()
optimiser = optim.Adam(model.parameters())
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.0
for x_batch, y_batch in tqdm(train_loader, disable=True):
y_predicted = model(x_batch)
 loss = loss_function(y_predicted, y_batch)
 optimiser.zero_grad()
loss.backward()
optimiser.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
val_predicted_fold = np.zeros(x_val_fold.size(0))
test_predicted_fold = np.zeros(len(test_X))
avg_val_loss =0.0
for i,(x_batch, y_batch)in enumerate(valid_loader):
y_predicted = model(x_batch ).detach()
avg_val_loss += loss_function(y_predicted, y_batch ).item() /len(valid_loader)
 val_predicted_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_predicted.cpu().numpy())[:,0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1,
train_epochs, avg_loss, avg_val_loss, elapsed_time))
test_predicted_fold = np.zeros(len(test_X))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_predicted_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = val_predicted_fold
test_preds += test_predicted_fold / len(splits )<compute_test_metric>
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<compute_test_metric>
def create_model(input_shape,n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights('.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dense(1024, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 10; batch_size = 8
checkpoint = ModelCheckpoint('.. /working/Densenet121.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=15)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 train_generator = My_Generator(train_x, train_y, 128, is_train=True)
 train_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)
 valid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)
 model = create_model(input_shape=(SIZE,SIZE,3),n_out=NUM_CLASSES)
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=1108, bsize=8, name='kappa'):

 with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom*0.5 /(denom + eps)+ categorical_crossentropy(y_true, y_pred)*0.5
import pandas as pd
import numpy as np
import operator
import re
import gc
import pickle
from keras.preprocessing.text import Tokenizer<define_variables>
for layer in model.layers:
layer.trainable = False
for i in range(-3,0):
model.layers[i].trainable = True
model.compile(
 loss='categorical_crossentropy',
optimizer=Nadam(1e-3))
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_y)) / float(128)) ,
epochs=30,
workers=WORKERS, use_multiprocessing=True,
verbose=1 )
np.random.seed(42 )<categorify>
for layer in model.layers:
layer.trainable = True
callbacks_list = [csv_logger, reduceLROnPlat]
model.compile(optimizer=Nadam(1e-4),
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)) ,
validation_data=valid_generator,
validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
workers=2, use_multiprocessing=True,
callbacks=callbacks_list )
def load_embed(file):
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)> 100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<feature_engineering>
model.save_weights(".. /working/Densenet121.h5" )
def index_embs(embeddings_index, word_index, NUM_WORDS, fileName):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(NUM_WORDS, embed_size))
for word, i in word_index.items() :
if i >= NUM_WORDS: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
np.save(fileName, embedding_matrix)
 print(fileName + " embedding matrix saved!")
 del(embeddings_index)
 del(word_index)
 del(embedding_matrix)
 gc.collect()<categorify>
submit = pd.read_csv('.. /input/recursion-cellular-image-classification/sample_submission.csv')
 model.load_weights('.. /working/Densenet121.h5')
 predicted = []
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<categorify>
for i, name in tqdm(enumerate(submit['id_code'])) :
path = os.path.join('.. /input/recursion-complete/test/test/', name+'_s1.jpeg')
 image = cv2.imread(path)
 score_predict = model.predict(( image[np.newaxis])/255)
 label_predict = np.argmax(score_predict)
 predicted.append(str(label_predict))
<categorify><EOS> 
submit['sirna'] = predicted
submit.to_csv('submission.csv', index=False)
 submit.head()