Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<correct_missing_values>
Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten
Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D
print(os.listdir(".. /input"))
train_dataset.dropna(inplace=True )<string_transform>
df_train = pd.read_csv(".. /input/train_labels.csv")
 id_label_map = {k:v for k,v in zip(df_train.id.values, df_train.label.values)}
df_train.head()
def remove_leading_truncuated_word(row):
text = row['text']
selected_text = row['selected_text']
first_selected_text_word = selected_text.lower().split() [0]
if(first_selected_text_word not in text.lower().split())and \ 
(len(first_selected_text_word)== 1):
return " ".join(selected_text.lower().split() [1:])
 else:
return selected_text
def modified_text_to_bert_input(text, tokenizer):
return tokenizer.convert_tokens_to_string(tokenizer.tokenize(text))<feature_engineering>
def get_id_from_file_path(file_path):
return file_path.split(os.path.sep)[-1].replace('.tif', '' )
tokenizer = BertTokenizer.from_pretrained('/kaggle/input/bertbaseuncased', max_len=192)
 train_dataset['text'] = train_dataset['text'].apply(modified_text_to_bert_input, args=(tokenizer,))
train_dataset['selected_text'] = train_dataset['selected_text'].apply(modified_text_to_bert_input, args=(tokenizer,))<prepare_x_and_y>
labeled_files = glob('.. /input/train/*.tif')
 test_files = glob('.. /input/test/*.tif' )
train_dataset['selected_text'] = train_dataset.apply(remove_leading_truncuated_word, axis=1)
 train_dataset['selected_text'].replace('', np.nan, inplace=True)
 train_dataset.dropna(inplace=True )<feature_engineering>
train, val = train_test_split(labeled_files, test_size=0.1, random_state=101010 )
def is_substring(row):
return row.loc['selected_text'] in row.loc['text']
train_dataset['is_substring'] = train_dataset.apply(is_substring, axis=1)
 train_dataset['is_substring'].value_counts()<count_values>
def get_model_classif_nasnet() :
inputs = Input(( 96, 96, 3))
base_model = NASNetMobile(include_top=False, input_shape=(96, 96, 3))
x = base_model(inputs)
 out1 = GlobalMaxPooling2D()(x)
 out2 = GlobalAveragePooling2D()(x)
 out3 = Flatten()(x)
 out = Concatenate(axis=-1 )([out1, out2, out3])
 out = Dropout(0.5 )(out)
 out = Dense(1, activation="sigmoid", name="3_" )(out)
 model = Model(inputs, out)
 model.compile(optimizer=Adam(0.0001), loss=binary_crossentropy, metrics=['acc'])
 model.summary()
return model
neutral_dataset = train_dataset[train_dataset['sentiment'] == 'neutral']
neutral_dataset['is_same'] = neutral_dataset['selected_text'] == neutral_dataset['text']
neutral_dataset['is_same'].value_counts()<filter>
model = get_model_classif_nasnet()
train_dataset = train_dataset[~(train_dataset['sentiment'] == 'neutral')]<define_variables>
batch_size=32
h5_path = "model.h5"
checkpoint = ModelCheckpoint(h5_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 history = model.fit_generator(
 data_gen(train, id_label_map, batch_size, augment=True),
validation_data=data_gen(val, id_label_map, batch_size),
epochs=2, verbose=1,
callbacks=[checkpoint],
steps_per_epoch=len(train)// batch_size,
validation_steps=len(val)// batch_size)
 batch_size=64
history = model.fit_generator(
 data_gen(train, id_label_map, batch_size, augment=True),
validation_data=data_gen(val, id_label_map, batch_size),
epochs=6, verbose=1,
callbacks=[checkpoint],
steps_per_epoch=len(train)// batch_size,
validation_steps=len(val)// batch_size)
 model.load_weights(h5_path )
def get_start_and_end_char_index(row):
text = row['text']
selected_text = row['selected_text']
start_idx = text.find(selected_text)
 end_idx = start_idx + len(selected_text)
 return start_idx, end_idx
def target_answer(row, tokenizer):
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in row['text']:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 start_idx = row['start_char_idx']
end_idx = row['end_char_idx']
start_position = char_to_word_offset[start_idx]
end_position = char_to_word_offset[end_idx - 1]
actual_text = " ".join(doc_tokens[start_position:end_position+1])
 tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = tokenizer.tokenize(token)
 for sub_token in sub_tokens:
tok_to_orig_index.append(i)
 all_doc_tokens.append(sub_token)
 tok_start_position = orig_to_tok_index[start_position]
if end_position < len(doc_tokens)- 1:
tok_end_position = orig_to_tok_index[end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_answer_text = tokenizer.tokenize(row['selected_text'])
 for new_start in range(tok_start_position, tok_end_position + 1):
for new_end in range(tok_end_position, tok_start_position - 1, -1):
text_span = all_doc_tokens[new_start : new_end + 1]
if text_span == tok_answer_text:
return new_start, new_end, tokenizer.convert_tokens_to_string(text_span)
 text_span = all_doc_tokens[tok_start_position : tok_end_position + 1]
return tok_start_position, tok_end_position, tokenizer.convert_tokens_to_string(text_span )<feature_engineering>
preds = []
ids = []
train_dataset[['start_char_idx', 'end_char_idx']] = train_dataset.apply(get_start_and_end_char_index, axis=1, result_type='expand' )<count_missing_values>
for batch in chunker(test_files, batch_size):
X = [preprocess_input(cv2.imread(x)) for x in batch]
ids_batch = [get_id_from_file_path(x)for x in batch]
X = np.array(X)
 preds_batch =(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
preds += preds_batch
ids += ids_batch
<count_values><EOS> 
df = pd.DataFrame({'id':ids, 'label':preds})
 df.to_csv("baseline_nasnet.csv", index=False)
 df.head()