Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<groupby>
import numpy as np
import pandas as pd
import cv2
import os
from glob import glob
import matplotlib.pyplot as plt
import random
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, roc_auc_score
from keras_preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation
from keras.layers import Conv2D, MaxPool2D
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from IPython.display import clear_output
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()<drop_column>
path = ".. /input/"
labels = pd.read_csv(path + 'train_labels.csv')
 train_path = path + 'train/'
test_path = path + 'test/'
new_data=new_data.drop('production_countries',axis=1 )<categorify>
df = pd.DataFrame({'path': glob(os.path.join(train_path,'*.tif')) })
 df['id'] = df.path.map(lambda x:(( x.split("n")[2].split('.')[0])[1:]))
df = df.merge(labels, on = "id")
 df.head(3 )
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'<drop_column>
def readImage(path):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 return rgb_img
d.remove(( 'u',414)) 
<concatenate>
IMG_SIZE = 196
BATCH_SIZE = 128
dd=[]
for i in d:
dd.append(i[0] )<concatenate>
test_list = os.listdir(test_path)
 train_list = os.listdir(train_path)
 print("There are " + str(len(train_list)) + " training examples.")
 print("There are " + str(len(test_list)) + " test examples." )
dd=[]
for i in d:
dd.append(i[0] )<drop_column>
df['label'] = df['label'].astype(str)
 train, valid = train_test_split(df, test_size=0.2, stratify = df['label'])
new_data=new_data.drop('spoken_languages',axis=1 )<feature_engineering>
def crop_centre(image, crop_length):
original_size = image.shape[0]
centre = original_size // 2
lower_bound = centre - crop_length // 2
upper_bound = centre + crop_length // 2
image = image[(lower_bound):(upper_bound),(lower_bound):(upper_bound)]
return image
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )<drop_column>
train_datagen = ImageDataGenerator(rescale=1./255,
vertical_flip = True,
horizontal_flip = True,
rotation_range=90,
zoom_range=0.2,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.05,
channel_shift_range=0.1)
 test_datagen = ImageDataGenerator(rescale = 1./255 )
new_data=new_data.drop('production_companies',axis=1 )<drop_column>
train_generator = train_datagen.flow_from_dataframe(dataframe = train,
directory = None,
x_col = 'path',
y_col = 'label',
target_size =(IMG_SIZE,IMG_SIZE),
class_mode = "binary",
batch_size=BATCH_SIZE,
seed = 110318,
shuffle = True )
new_data=new_data.drop('crew',axis=1 )<feature_engineering>
valid_generator = test_datagen.flow_from_dataframe(dataframe = valid,
directory = None,
x_col = 'path',
y_col = 'label',
target_size =(IMG_SIZE,IMG_SIZE),
class_mode = 'binary',
batch_size = BATCH_SIZE,
shuffle = False )
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )<feature_engineering>
dropout_fc = 0.5
conv_base = ResNet50(weights = 'imagenet', include_top = False, input_shape =(IMG_SIZE,IMG_SIZE,3))
my_model = Sequential()
my_model.add(conv_base)
 my_model.add(Flatten())
 my_model.add(Dense(256, use_bias=False))
my_model.add(BatchNormalization())
 my_model.add(Activation("relu"))
my_model.add(Dropout(dropout_fc))
my_model.add(Dense(1, activation = "sigmoid"))
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )<feature_engineering>
conv_base.Trainable=True
set_trainable=False
for layer in conv_base.layers:
if layer.name == 'res5a_branch2a':
set_trainable = True
if set_trainable:
layer.trainable = True
else:
layer.trainable = False
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']<count_values>
my_model.compile(optimizers.Adam(0.001), loss = "binary_crossentropy", metrics = ["accuracy"] )
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)] 
<categorify>
train_step_size = train_generator.n // train_generator.batch_size
valid_step_size = valid_generator.n // valid_generator.batch_size
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['_releaseYear_popularity_ratio2'] = new_data['popularity'] / new_data['release_year']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
<feature_engineering>
earlystopper = EarlyStopping(monitor='val_loss', patience=3, verbose=2, restore_best_weights=True)
 reduce = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.1 )
<feature_engineering>
history = my_model.fit_generator(train_generator,
steps_per_epoch = train_step_size,
epochs = 10,
validation_data = valid_generator,
validation_steps = valid_step_size,
callbacks = [reduce, earlystopper],
verbose = 2)
<split>
roc_validation_generator = ImageDataGenerator(rescale=1./255 ).flow_from_dataframe(valid,
x_col = 'path',
y_col = 'label',
target_size =(IMG_SIZE,IMG_SIZE),
class_mode = 'binary',
batch_size = BATCH_SIZE,
shuffle = False)
 predictions = my_model.predict_generator(roc_validation_generator, steps=len(roc_validation_generator), verbose=2)
 false_positive_rate, true_positive_rate, threshold = roc_curve(roc_validation_generator.classes, predictions)
 area_under_curve = auc(false_positive_rate, true_positive_rate)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))
plt.xlabel('False positive rate')
 plt.ylabel('True positive rate')
 plt.title('ROC curve')
 plt.legend(loc='best')
 plt.savefig('ROC_PLOT.png', bbox_inches='tight')
 plt.show()
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]<train_model>
testdf = pd.DataFrame({'path': glob(os.path.join(test_path, '*.tif')) })
 testdf['id'] = testdf.path.map(lambda x:(x.split("/")[3].split('.')[0]))
testdf.head(3 )
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
<train_model>
tta_datagen = ImageDataGenerator(rescale=1./255,
vertical_flip = True,
horizontal_flip = True,
rotation_range=90,
zoom_range=0.2,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.05,
channel_shift_range=0.1 )
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf)) 
<drop_column>
tta_steps = 5
submission = pd.DataFrame()
for index in range(0, len(testdf)) :
data_frame = pd.DataFrame({'path': testdf.iloc[index,0]}, index=[index])
 data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split('.')[0])
 img_path = data_frame.iloc[0,0]
test_img = cv2.imread(img_path)
 test_img = cv2.resize(test_img,(IMG_SIZE,IMG_SIZE))
test_img = np.expand_dims(test_img, axis = 0)
 predictionsTTA = []
for i in range(0, tta_steps):
preds = my_model.predict_generator(tta_datagen.flow_from_dataframe(dataframe = data_frame,
directory = None,
x_col = 'path',
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = None,
batch_size = 1,
shuffle = False), steps = 1)
 predictionsTTA.append(preds)
 clear_output()
prediction_entry = np.array(np.mean(predictionsTTA, axis=0))
data_frame['label'] = prediction_entry
submission = pd.concat([submission, data_frame[['id', 'label']]])
<predict_on_test><EOS> 
submission.to_csv('submission.csv', index=False, header=True )