Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<data_type_conversions>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
y_test = y_test.astype(int)
 y_test<prepare_x_and_y>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
y_pseudo_label = np.append(y, y_test, axis = 0)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
list_sentences_train_pseudo = np.append(list_sentences_train, list_sentences_test, axis = 0 )<string_transform>
df_trn=pd.read_csv(train_lbl )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train_pseudo))
list_tokenized_train_pseudo = tokenizer.texts_to_sequences(list_sentences_train_pseudo)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t_pseudo = pad_sequences(list_tokenized_train_pseudo, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<compute_test_metric>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))<feature_engineering>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
data.show_batch(rows=5, figsize=(12,9))
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
from sklearn.metrics import roc_auc_score
model.fit(X_t_pseudo, y_pseudo_label, batch_size=32, epochs=2, validation_split=0.1);<save_to_csv>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
y_test_final = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test_final
sample_submission.to_csv('submission.csv', index=False )<set_options>
from torchvision.models import *
warnings.filterwarnings("ignore")
<load_from_csv>
learn = create_cnn(
 data,
densenet169,
path='.',
metrics=[auc_score],
ps=0.5
)
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip' ).fillna(' ')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip' ).fillna(' ' )<concatenate>
learn.fit_one_cycle(1,3e-2)
 learn.recorder.plot()
learn.recorder.plot_losses()
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text])
 class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']<feature_engineering>
learn.unfreeze()
learn.lr_find()
word_vec = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
max_features=30000)
 char_vec = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 3),
max_features=30000)
 vec1 = make_union(word_vec, char_vec)
 vec1.fit(all_text)
 train_features = vec1.transform(train_text)
 test_features = vec1.transform(test_text )<string_transform>
learn.fit_one_cycle(20,slice(1e-4,1e-3))
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s):
return re_tok.sub(r' \1 ', s ).split()
vec2 = TfidfVectorizer(ngram_range=(1,2),
tokenizer=tokenize,
min_df=3,
max_df=0.9,
strip_accents='unicode',
use_idf=1,
smooth_idf=1,
sublinear_tf=1)
 train_term_doc = vec2.fit_transform(train_text)
 test_term_doc = vec2.transform(test_text )<compute_train_metric>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
scores1 = []
sub1 = pd.DataFrame.from_dict({'id': test['id']})
 for class_name in class_names:
train_target = train[class_name]
classifier = LogisticRegression(C = 0.1, solver='saga', class_weight='balanced')
 cv_score = np.mean(cross_val_score(classifier, train_features, train_target, cv=3, scoring='roc_auc'))
scores1.append(cv_score)
 print('CV score for class {} is {:.4%}'.format(class_name, cv_score))
classifier.fit(train_features, train_target)
 sub1[class_name] = classifier.predict_proba(test_features)[:, 1] 
<compute_train_metric>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
train_x = train_term_doc
test_x = test_term_doc
def pr(y_i, y):
p = train_x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 def get_mdl(y):
y = y.values
r = np.log(pr(1,y)/ pr(0,y))
m = LogisticRegression(C=4, solver='liblinear', dual=True)
 x_nb = train_x.multiply(r)
 return m.fit(x_nb, y), r
scores2 = []
sub2 = pd.DataFrame.from_dict({'id': test['id']})
 for i, j in enumerate(class_names):
m,r = get_mdl(train[j])
 sub2[j] = m.predict_proba(test_x.multiply(r)) [:,1]
y_pred = m.predict_proba(train_x.multiply(r)) [:,1]
print('fit score for class {} is {:.4%}'.format(j, roc_auc_score(train_target, y_pred)))
 scores2.append(roc_auc_score(train_target, y_pred))<categorify>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
sub = sub1.copy()
for i, j in enumerate(class_names):
pb = np.array([scores1[i], scores2[i]])
 weights = lambda x: x/sum(x)
 w = weights(pb)
 print(w)
 sub[j] = sub1[j]*w[0] + sub2[j]*w[1]<save_to_csv>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
sub.to_csv('submission.csv', index=False )<save_to_csv>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
sub.to_csv('submission.csv', index=False )<set_options>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
tqdm.pandas()<load_from_csv>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<count_missing_values><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )