Kaggle competition: Recursion Cellular Image Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: recursion-cellular-image-classification<feature_engineering>
SIZE = 224
NUM_CLASSES = 1108
train_csv = pd.read_csv(".. /input/recursion-cellular-image-classification/train.csv")
 test_csv = pd.read_csv(".. /input/recursion-cellular-image-classification/test.csv")
 sub = pd.read_csv(".. /input/recursion-cellular-keras-densenet/submission.csv" )
def statistical_features(df):
stat_features = pd.DataFrame()
stat_features['txt_len'] = df['question_text'].progress_apply(lambda x: len(str(x)))
 stat_features['word_count'] = df['question_text'].progress_apply(lambda x: len(str(x ).split(" ")))
 stat_features['!_count'] = df['question_text'].progress_apply(lambda x: x.count('!'))
stat_features['?_count'] = df['question_text'].progress_apply(lambda x: x.count('?'))
stat_features['upper_word_count'] = df['question_text'].progress_apply(lambda x: len([x for x in x.split() if x.isupper() ]))
stat_features['unique_word_count'] = df['question_text'].progress_apply(lambda x: len(set(x.split())))
 return stat_features<split>
np.stack([train_csv.plate.values[train_csv.sirna == i] for i in range(10)] ).transpose()
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<load_pretrained>
train_csv.loc[train_csv.sirna==0,'plate'].value_counts()
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
print("Extrayendo GloVe embedding...")
 embed_glove = load_embed(glove )<train_model>
plate_groups = np.zeros(( 1108,4), int)
 for sirna in range(1108):
grp = train_csv.loc[train_csv.sirna==sirna,:].plate.value_counts().index.values
assert len(grp)== 3
plate_groups[sirna,0:3] = grp
plate_groups[sirna,3] = 10 - grp.sum()
plate_groups[:10,:]
<feature_engineering>
all_test_exp = test_csv.experiment.unique()
group_plate_probs = np.zeros(( len(all_test_exp),4))
for idx in range(len(all_test_exp)) :
preds = sub.loc[test_csv.experiment == all_test_exp[idx],'sirna'].values
pp_mult = np.zeros(( len(preds),1108))
pp_mult[range(len(preds)) ,preds] = 1
sub_test = test_csv.loc[test_csv.experiment == all_test_exp[idx],:]
assert len(pp_mult)== len(sub_test)
 for j in range(4):
mask = np.repeat(plate_groups[np.newaxis, :, j], len(pp_mult), axis=0)== \
np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)
 group_plate_probs[idx,j] = np.array(pp_mult)[mask].sum() /len(pp_mult )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<categorify>
pd.DataFrame(group_plate_probs, index = all_test_exp )
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Se agregaron {count} palabras al embedding" )<concatenate>
exp_to_group = group_plate_probs.argmax(1)
 print(exp_to_group )
df = pd.concat([train_df ,test_df])
 vocab = build_vocab(df['question_text'] )<compute_test_metric>
def create_model(input_shape,n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dense(1024, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
print("GloVe v2: ")
 add_lower(embed_glove, vocab)
<feature_engineering>
model = create_model(input_shape=(SIZE,SIZE,3),n_out=NUM_CLASSES )
train_df['treated_question'] = train_df['question_text'].progress_apply(lambda x: x.lower())
 train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df['treated_question'] = test_df['question_text'].progress_apply(lambda x: x.lower())
 test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: correct_spelling(x, mispell_dict))<define_variables>
model.load_weights('.. /input/recursion-cellular-keras-densenet/Densenet121.h5' )
embed_size = 300
max_features = 95000
max_len = 100<string_transform>
predicted = []
for i, name in tqdm(enumerate(test_csv['id_code'])) :
path1 = os.path.join('.. /input/recursion-cellular-image-classification-224-jpg/test/test/', name+'_s1.jpeg')
 image1 = cv2.imread(path1)
 score_predict1 = model.predict(( image1[np.newaxis])/255)
 path2 = os.path.join('.. /input/recursion-cellular-image-classification-224-jpg/test/test/', name+'_s2.jpeg')
 image2 = cv2.imread(path2)
 score_predict2 = model.predict(( image2[np.newaxis])/255)
 predicted.append(0.5*(score_predict1 + score_predict2))
def make_data(X, X_test):
t = Tokenizer(num_words=max_features, filters='', oov_token='<OOV>')
 t.fit_on_texts(X)
 X = t.texts_to_sequences(X)
 X_test = t.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 return X, X_test, t.word_index<prepare_x_and_y>
def select_plate_group(pp_mult, idx):
sub_test = test_csv.loc[test_csv.experiment == all_test_exp[idx],:]
assert len(pp_mult)== len(sub_test)
 mask = np.repeat(plate_groups[np.newaxis, :, exp_to_group[idx]], len(pp_mult), axis=0)!= \
np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)
 pp_mult[mask] = 0
return pp_mult
X, X_test, word_index = make_data(train_df['treated_question'], test_df['treated_question'] )<split>
for idx in range(len(all_test_exp)) :
indices =(test_csv.experiment == all_test_exp[idx])
 preds = predicted[indices,:].copy()
preds = select_plate_group(preds, idx)
 sub.loc[indices,'sirna'] = preds.argmax(1 )
y = train_df['target'].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=42 )<normalization>
( sub.sirna == pd.read_csv(".. /input/recursion-cellular-keras-densenet/submission.csv" ).sirna ).mean()
<prepare_x_and_y><EOS> 
sub.to_csv('.. /working/submission.csv', index=False, columns=['id_code','sirna'] )