Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<sort_values>
print(os.listdir(".. /input"))
rev=rev.sort_values(ascending=False)[:10]<filter>
df_train = pd.read_csv(".. /input/histopathologic-cancer-detection/train_labels.csv")
 id_label_map = {k:v for k,v in zip(df_train.id.values, df_train.label.values)}
df_train.head()
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()<define_variables>
def get_id_from_file_path(file_path):
return file_path.split(os.path.sep)[-1].replace('.tif', '' )
count_rev=along_co[['production_countries','revenue']]<prepare_output>
labeled_files = glob('.. /input/histopathologic-cancer-detection/train/*.tif')
 test_files = glob('.. /input/histopathologic-cancer-detection/test/*.tif' )
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()<load_pretrained>
train, val = train_test_split(labeled_files, test_size=0.1, random_state=101010 )
import geopandas as ge 
<load_from_csv>
def get_model_classif_nasnet() :
inputs = Input(( 96, 96, 3))
x1 = layers.Conv2D(32,3,padding='same' )(inputs)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.MaxPool2D(2,2 )(x1)
 x2 = layers.Conv2D(64,3,padding='same' )(x1)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 residual_x1 = layers.Conv2D(64,1,strides=1,padding='same' )(x1)
 x2 = layers.add([x2,residual_x1])
 x2 = layers.MaxPool2D(2,2 )(x2)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2)
 x2_s = layers.BatchNormalization()(x2_s)
 x2_s = layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.add([x2_s,x2])
 x2_s = layers.MaxPool2D(2,2 )(x2_s)
 x3 = layers.Conv2D(128,3,padding='same' )(x2)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 residual_x2 = layers.Conv2D(128,1,strides=1,padding='same' )(x2)
 x3 = layers.add([residual_x2,x3])
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.add([x3,x3_x3])
 x3_x3 = layers.MaxPool2D(2,2 )(x3_x3)
 concetenated_1 = layers.concatenate([x3_x3,x2_s])
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(concetenated_1)
 x3_s = layers.BatchNormalization()(x3_s)
 x3_s = layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.add([x3_s,x3_x3])
 x3_s = layers.MaxPool2D(2,2 )(x3_s)
 x4 = layers.Conv2D(256,3,padding='same' )(x3_x3)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 residual_x3 = layers.Conv2D(256,1,strides=1,padding='same' )(x3_x3)
 x4 = layers.add([residual_x3,x4])
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.add([x4,x4_x4])
 x4_x4 = layers.MaxPool2D(2,2 )(x4_x4)
 concetenated_2 = layers.concatenate([x4_x4,x3_s])
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(concetenated_2)
 x4_s = layers.BatchNormalization()(x4_s)
 x4_s = layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.add([x4_s,x4_x4])
 x4_s = layers.MaxPool2D(2,2 )(x4_s)
 x5 = layers.Conv2D(512,3,padding='same' )(x4_x4)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 residual_x4 = layers.Conv2D(512,1,strides=1,padding='same' )(x4_x4)
 x5 = layers.add([residual_x4,x5])
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.add([x5,x5_x5])
 x5_x5 = layers.MaxPool2D(2,2 )(x5_x5)
 concetenated_3 = layers.concatenate([x5_x5,x4_s])
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(concetenated_3)
 x5_s = layers.BatchNormalization()(x5_s)
 x5_s = layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.add([x5_s,x5_x5])
 x = layers.GlobalAveragePooling2D()(x5_s)
 x = layers.Dense(64 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Activation('relu' )(x)
 x = layers.Dropout(0.2 )(x)
 output_tensor = layers.Dense(1,activation='sigmoid' )(x)
 model = Model(inputs,output_tensor)
 model.compile(optimizer=Adam(0.01), loss=binary_crossentropy, metrics=['acc'])
 model.summary()
return model
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))<import_modules>
def get_model_classif_nasnet_1() :
inputs = Input(( 96, 96, 3))
x1 = layers.Conv2D(32,3,padding='same' )(inputs)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(inputs)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(x1_s)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(x1_s)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 concetenated_0 = layers.concatenate([x1,x1_s])
 x2 = layers.Conv2D(64,3,padding='same' )(concetenated_0)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 residual_concetenated_0 = layers.Conv2D(64,1,strides=1,padding='same' )(concetenated_0)
 x2 = layers.add([x2,residual_concetenated_0])
 concetenates_x2_x1_s = layers.concatenate([x2,x1_s])
 x2 = layers.MaxPool2D(2,2 )(concetenates_x2_x1_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2)
 x2_s = layers.BatchNormalization()(x2_s)
 x2_s = layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.Conv2D(96,1,strides=1,padding='same' )(x2_s)
 x2_s = layers.add([x2_s,x2])
 x2_s = layers.MaxPool2D(2,2 )(x2_s)
 x3 = layers.Conv2D(128,3,padding='same' )(x2)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 residual_x2 = layers.Conv2D(128,1,strides=1,padding='same' )(x2)
 x3 = layers.add([residual_x2,x3])
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.add([x3,x3_x3])
 x3_x3 = layers.MaxPool2D(2,2 )(x3_x3)
 concetenated_1 = layers.concatenate([x3_x3,x2_s])
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(concetenated_1)
 x3_s = layers.BatchNormalization()(x3_s)
 x3_s = layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.add([x3_s,x3_x3])
 x3_s = layers.MaxPool2D(2,2 )(x3_s)
 x4 = layers.Conv2D(256,3,padding='same' )(x3_x3)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 residual_x3 = layers.Conv2D(256,1,strides=1,padding='same' )(x3_x3)
 x4 = layers.add([residual_x3,x4])
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.add([x4,x4_x4])
 x4_x4 = layers.MaxPool2D(2,2 )(x4_x4)
 concetenated_2 = layers.concatenate([x4_x4,x3_s])
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(concetenated_2)
 x4_s = layers.BatchNormalization()(x4_s)
 x4_s = layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.add([x4_s,x4_x4])
 x4_s = layers.MaxPool2D(2,2 )(x4_s)
 x5 = layers.Conv2D(512,3,padding='same' )(x4_x4)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 residual_x4 = layers.Conv2D(512,1,strides=1,padding='same' )(x4_x4)
 x5 = layers.add([residual_x4,x5])
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.add([x5,x5_x5])
 x5_x5 = layers.MaxPool2D(2,2 )(x5_x5)
 concetenated_3 = layers.concatenate([x5_x5,x4_s])
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(concetenated_3)
 x5_s = layers.BatchNormalization()(x5_s)
 x5_s = layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.add([x5_s,x5_x5])
 x = layers.GlobalAveragePooling2D()(x5_s)
 x = layers.Dense(64 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Activation('relu' )(x)
 x = layers.Dropout(0.3 )(x)
 output_tensor = layers.Dense(1,activation='sigmoid' )(x)
 model = Model(inputs,output_tensor)
 model.compile(optimizer=Adam(0.001), loss='binary_crossentropy', metrics=['acc'])
 model.summary()
return model
from mpl_toolkits.axes_grid1 import make_axes_locatable 
<count_values>
model = get_model_classif_nasnet_1()
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()<categorify>
model.load_weights('.. /input/model-3/model_3.h5')
 batch_size=128
h5_path = "model_4.h5"
checkpoint = ModelCheckpoint(h5_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 history = model.fit_generator(
 data_gen(train, id_label_map, batch_size, augment=True),
validation_data=data_gen(val, id_label_map, batch_size),
epochs=5, verbose=1,
callbacks=[checkpoint],
steps_per_epoch=len(train)// batch_size,
validation_steps=len(val)// batch_size)
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland' 
<drop_column>
preds = []
ids = []
mn=mn.drop(10)
 mn=mn.drop(34 )<feature_engineering>
model.load_weights(h5_path)
 for batch in chunker(test_files, batch_size):
X = [preprocess_input(cv2.imread(x)) for x in batch]
ids_batch = [get_id_from_file_path(x)for x in batch]
X = np.array(X)
 preds_batch =(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
preds += preds_batch
ids += ids_batch
<feature_engineering><EOS> 
df = pd.DataFrame({'id':ids, 'label':preds})
 df.to_csv("baseline_nasnet.csv", index=False)
 df.head()