Kaggle competition: Pneumonia Texture Analysis
<SOS> metric: auc Kaggle data source: pneumonia-texture-analysis<prepare_x_and_y>
%matplotlib inline
data_dir = '.. /input/'
y_train_tfidf = train_pruned['target']
y_val_tfidf = val_pruned['target']<compute_train_metric>
def categories_to_indicators(in_df):
new_df = in_df.copy()
new_df['IsMale'] = in_df['PatientSex'].map(lambda x: 'M' in x ).astype(float)
 new_df['IsAP'] = in_df['ViewPosition'].map(lambda x: 'AP' in x ).astype(float)
 return new_df.drop(['PatientSex', 'ViewPosition'], axis=1)
 full_train_df = categories_to_indicators(pd.read_csv(os.path.join(data_dir, 'train_all.csv')))
 full_stack = imread(os.path.join(data_dir, 'train.tif'))
full_train_df['image'] = full_train_df['slice_idx'].map(lambda x: full_stack[x])
 full_train_df.sample(3 )
X_train_keyword = csr_matrix(train_pruned.iloc[:, 3:] ).tocsr()
X_val_keyword = csr_matrix(val_pruned.iloc[:, 3:] ).tocsr()
X_test_keyword = csr_matrix(test_pruned.iloc[:, 2:] ).tocsr()<compute_train_metric>
submission_test_df = categories_to_indicators(pd.read_csv(os.path.join(data_dir, 'test_info.csv')))
 test_stack = imread(os.path.join(data_dir, 'test.tif'))
submission_test_df['image'] = submission_test_df['slice_idx'].map(lambda x: full_stack[x])
 submission_test_df.sample(3 )
mb_clf = MB()
mb_clf.fit(X_train_tfidf, y_train_tfidf)
 pred = mb_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, pred))
stratified_cv(mb_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<train_on_grid>
def fit_and_score(in_model, feature_maker, rescale=True):

 full_features = feature_maker(full_train_df)
 full_labels = full_train_df['opacity']
submission_feat = feature_maker(submission_test_df)
 train_feat, valid_feat, train_lab, valid_lab = train_test_split(full_features,
full_labels,
test_size=0.25,
random_state=2018)
 if rescale:
feature_scaler = RobustScaler()
train_feat = feature_scaler.fit_transform(train_feat)
 valid_feat = feature_scaler.transform(valid_feat)
 submission_feat = feature_scaler.transform(submission_feat)
 in_model.fit(train_feat, train_lab)
 predictions = in_model.predict_proba(valid_feat)[:, 1]
predicted_class = predictions>0.5
tpr, fpr, _ = roc_curve(valid_lab, predictions)
 auc = roc_auc_score(valid_lab, predictions)
 fig,(ax1, ax2)= plt.subplots(1, 2, figsize=(20, 10))
ax1.plot(tpr, fpr, 'r.-', label='Prediction(AUC:{:2.2f})'.format(auc))
ax1.plot(tpr, tpr, 'k-', label='Random Guessing')
 ax1.legend()
ax1.set_title('ROC Curve')
 print(classification_report(valid_lab, predicted_class, target_names=['opacity', 'no opacity']))
sns.heatmap(confusion_matrix(valid_lab, predicted_class),
annot=True, fmt='4d', ax=ax2)
 ax2.set_xlabel('Prediction')
 ax2.set_ylabel('Actual Value')
 ax2.set_title('Confusion Matrix({:.1%})'.format(accuracy_score(valid_lab, predicted_class)))
 sub_df = submission_test_df[['tile_id']].copy()
sub_df['opacity'] = in_model.predict_proba(submission_feat)[:, 1]
sub_df[['tile_id', 'opacity']].to_csv(
 'm-{model}-f-{features}-s-{auc:2.0f}.csv'.format(
 model=in_model.__class__.__name__,
features=feature_maker.__name__,
auc=100*auc, 
), index=False )
mb_clf_augmented = MB()
mb_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 pred = mb_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(mb_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<compute_train_metric>
dum_model = DummyClassifier(strategy='stratified', random_state=2018)
 def justage(in_df):
return in_df[['PatientAge']].values
fit_and_score(
 dum_model,
justage
)
mb_clf_keyword = MB()
mb_clf_keyword.fit(X_train_keyword, y_train_tfidf)
 pred = mb_clf_keyword.predict(X_train_keyword)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(mb_clf_keyword, X_train_keyword, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<train_model>
knn_model = KNeighborsClassifier(1)
 def table_features(in_df):
return in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].values
fit_and_score(
 knn_model,
table_features
)
<train_on_grid>
def basic_image_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
out_df['Mean_Intensity'] = in_df['image'].map(np.mean)
 out_df['Std_Intensity'] = in_df['image'].map(np.std)
 return out_df.values
knn_model = KNeighborsClassifier(2)
 fit_and_score(
 knn_model,
basic_image_features
)
svm_clf = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf.fit(X_train_tfidf, y_train_tfidf)
 pred = svm_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<train_on_grid>
lr_model = LogisticRegression(solver='lbfgs', random_state=2018)
 fit_and_score(
 lr_model,
basic_image_features
)
svm_clf_augmented = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 pred = svm_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<compute_train_metric>
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
basic_image_features
)
svm_clf_keyword = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_keyword.fit(X_train_keyword, y_train_tfidf)
 pred = svm_clf_keyword.predict(X_train_keyword)
 print(classification_report(y_train_tfidf,pred))
stratified_cv(svm_clf_keyword, X_train_keyword, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<predict_on_test>
nb_model = GaussianNB()
fit_and_score(
 nb_model,
basic_image_features
)
svm_train_preds = svm_clf_keyword.predict(X_train_keyword)
 svm_val_preds = svm_clf_keyword.predict(X_val_keyword)
 svm_test_preds = svm_clf_keyword.predict(X_test_keyword )<train_on_grid>
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
basic_image_features
)
rf_clf = RandomForestClassifier(n_estimators = 100, random_state = RANDOM_STATE)
 rf_clf.fit(X_train_tfidf, y_train_tfidf)
 rf_preds = rf_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, rf_preds))
stratified_cv(rf_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<compute_train_metric>
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
texture_features
)
rf_clf_augmented = RandomForestClassifier(n_estimators = 100, random_state = RANDOM_STATE)
 rf_clf_augmented.fit(X_train_augmented, y_train_tfidf)
 rf_preds_augmented = rf_clf_augmented.predict(X_train_augmented)
 print(classification_report(y_train_tfidf, rf_preds))
stratified_cv(rf_clf_augmented, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<predict_on_test>
c_model = models.Sequential()
c_model.add(PTModel(include_top=False,
input_shape=full_train_df['image'].iloc[0].shape+(3,),
weights='imagenet'))
c_model.add(layers.GlobalAvgPool2D())
 def vgg_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
full_image_stack = np.stack(in_df['image'], 0)
 color_image_stack = np.stack([full_image_stack, full_image_stack, full_image_stack], -1 ).astype(float)
 pp_color_image_stack = preprocess_input(color_image_stack)
 vgg_features = c_model.predict(pp_color_image_stack)
 return np.concatenate([out_df.values, vgg_features], 1 )
rf_train_preds = rf_clf_augmented.predict(X_train_augmented)
 rf_val_preds = rf_clf_augmented.predict(X_val_augmented)
 rf_test_preds = rf_clf_augmented.predict(X_test_augmented )<compute_train_metric>
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
vgg_features
)
lr_clf = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf.fit(X_train_tfidf, y_train_tfidf)
 lr_preds = lr_clf.predict(X_train_tfidf)
 print(classification_report(y_train_tfidf, lr_preds))
stratified_cv(lr_clf, X_train_tfidf, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<train_on_grid>
xg_model = XGBClassifier()
fit_and_score(
 xg_model,
vgg_features
)
lr_clf = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf.fit(X_train_augmented, y_train_tfidf)
 lr_preds = lr_clf.predict(X_train_augmented)
 print(classification_report(y_train_tfidf, lr_preds))
stratified_cv(lr_clf, X_train_augmented, y_train_tfidf, random_state=RANDOM_STATE, shuffle=True )<set_options>
trn_image, vld_image, trn_label , vld_label = train_test_split(full_train_df['image'],
full_train_df['opacity'],
test_size=0.25,
random_state=2018)
 trn_image = np.stack(trn_image, 0)
 vld_image = np.stack(vld_image, 0 )
tf.config.set_visible_devices([], 'GPU')
 visible_devices = tf.config.get_visible_devices()<string_transform>
out_model = models.Sequential()
out_model.add(layers.Reshape(( 64, 64, 1), input_shape=trn_image.shape[1:]))
out_model.add(layers.Conv2D(16,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(32,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(64,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.Conv2D(128,(3, 3), padding='valid', activation='relu'))
out_model.add(layers.MaxPool2D(( 2, 2)))
 out_model.add(layers.GlobalAveragePooling2D())
 out_model.add(layers.Dense(32, activation='relu'))
out_model.add(layers.Dense(1, activation='sigmoid'))
out_model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['binary_accuracy'])
 out_model.summary()
train_corpus = train_pruned['text'].apply(lambda x: x.split() ).to_list()
val_corpus = val_pruned['text'].apply(lambda x: x.split() ).to_list()
test_corpus = test_pruned['text'].apply(lambda x: x.split() ).to_list()<define_variables>
fit_results = out_model.fit(trn_image, trn_label,
validation_data=(vld_image, vld_label),
epochs=100)
 clear_output()
corpus = train_corpus + val_corpus + test_corpus<prepare_x_and_y>
def custom_cnn_features(in_df):
out_df = in_df[['PatientAge', 'opacity_prior', 'IsMale', 'IsAP']].copy()
full_image_stack = np.stack(in_df['image'], 0)
 model_pred = out_model.predict(full_image_stack)
 return np.concatenate([out_df.values, model_pred], 1 )
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
 train_sequences = tokenizer.texts_to_sequences(train_corpus)
 val_sequences = tokenizer.texts_to_sequences(val_corpus)
 test_sequences = tokenizer.texts_to_sequences(test_corpus)
 word_index = tokenizer.word_index
train_max_length = max([len(x)for x in train_sequences])
 val_max_length = max([len(x)for x in val_sequences])
 test_max_length = max([len(x)for x in test_sequences])
 max_length = max(train_max_length, val_max_length, test_max_length)
 X_train_pad = pad_sequences(train_sequences, maxlen=max_length, padding="post")
 y_train = train_pruned['target']
X_val_pad = pad_sequences(val_sequences, maxlen=max_length, padding="post")
 y_val = val_pruned['target']
X_test_pad = pad_sequences(test_sequences, maxlen=max_length, padding="post")
 vocab = np.array(list(tokenizer.word_index.keys()))
vocab_size = len(tokenizer.word_index)+ 1<define_variables>
rf_model = RandomForestClassifier(random_state=2018)
 fit_and_score(
 rf_model,
custom_cnn_features
)
EMBEDDING_DIM = 30<choose_model_class>
svm_model = SVC(probability=True)
 fit_and_score(
 svm_model,
custom_cnn_features
)
def ConvNet(max_sequence_length, num_words, embedding_dim, labels_index):
embedding_layer = Embedding(num_words,
embedding_dim,
input_length=max_sequence_length)
 sequence_input = Input(shape=(max_sequence_length,), dtype='int32')
 embedded_sequences = embedding_layer(sequence_input)
 print()
convs = []
filter_sizes = [2, 3, 4, 5, 6]
for filter_size in filter_sizes:
l_conv = Conv1D(filters=30,
kernel_size=filter_size,
activation='relu' )(embedded_sequences)
 l_conv = Dropout(0.2 )(l_conv)
 l_pool = GlobalMaxPooling1D()(l_conv)
 convs.append(l_pool)
 l_merge = concatenate(convs, axis=1)
 x = Dense(30, activation='relu' )(l_merge)
 x = Dropout(0.2 )(x)
 preds = Dense(labels_index, activation='sigmoid', kernel_regularizer=l1_l2(0.01, 0.01))(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc', Precision() , Recall() ])
 model.summary()
return model<choose_model_class>
xg_model = XGBClassifier()
fit_and_score(
 xg_model,
custom_cnn_features
)
<train_model><EOS> 
!ls *.csv