Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<feature_engineering>
plt.rcParams['figure.figsize'] =(20,10)
 def append_ext(fn):
return fn+".tif"
print(os.listdir(".. /input"))
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]<sort_values>
seed(1)
 set_random_seed(1 )
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]<count_values>
traindf=pd.read_csv(".. /input/train_labels.csv",dtype=str)
 train_size = 180000
traindf = traindf.sort_values(by=['label','id'])
 traindf = traindf.iloc[:int(train_size/2)].append(traindf.iloc[-int(train_size/2):])
 testdf=pd.read_csv(".. /input/sample_submission.csv",dtype=str)
 traindf["id"]=traindf["id"].apply(append_ext)
 testdf["id"]=testdf["id"].apply(append_ext)
 datagen=ImageDataGenerator(rescale=1./255.,validation_split=0.25 )
old_title=new_data['title'].value_counts()<prepare_x_and_y>
batch_size = 64
train_generator=datagen.flow_from_dataframe(
 dataframe=traindf,
directory=".. /input/train/",
x_col="id",
y_col="label",
subset="training",
batch_size=batch_size,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(96, 96)
)
 valid_generator=datagen.flow_from_dataframe(
 dataframe=traindf,
directory=".. /input/train/",
x_col="id",
y_col="label",
subset="validation",
batch_size=batch_size,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(96, 96)
)
 test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=testdf,
directory=".. /input/test/",
x_col="id",
y_col=None,
batch_size=batch_size,
seed=42,
shuffle=False,
class_mode=None,
target_size=(96, 96)
 )
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index<feature_engineering>
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
 return auc
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )<filter>
def make_model(model_choice, input_tensor):

 base_model = model_choice
x = base_model(input_tensor)
 out = Dense(1, activation="softmax" )(x)
 model = Model(input_tensor, out)
 model.summary()
return model
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]<sort_values>
input_tensor = Input(( 96, 96, 3))
VGG19_model=make_model(VGG19(include_top=False, pooling='avg', input_tensor=input_tensor), input_tensor )
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]<count_values>
best_model_weights = './base.model'
checkpoint = ModelCheckpoint(
 best_model_weights,
monitor = 'val_loss',
save_best_only = True,
mode='min',
verbose=1,
save_weights_only=False,
period = 1)
 earlyStop = EarlyStopping(
 monitor = 'val_loss',
min_delta=0.01,
patience = 5,
verbose = 1,
mode = 'min')
 tensorBoard = TensorBoard(
 log_dir = './logs',
histogram_freq=0,
batch_size=batch_size,
write_graph=True,
write_grads=True,
write_images=False)
 reduce = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=3,
verbose=1,
mode='min' 
)
 csvlogger = CSVLogger(filename='training_csv.log',
separator=',',
append=False)
 callbacks = [checkpoint, tensorBoard, csvlogger, reduce]
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )<filter>
start=datetime.now()
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size
train[train['title'].str.contains('Furious')]['title'] 
<filter>
VGG19_model.compile(optimizer=SGD(lr=1e-4, momentum=0.99), loss=binary_crossentropy, metrics=['accuracy', auc])
 history = VGG19_model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=6,
callbacks=callbacks 
)
 end = datetime.now()
print("Elapsed time:",end-start )
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']<drop_column>
VGG19_model.load_weights(best_model_weights)
 VGG19_model.evaluate_generator(generator=valid_generator,
steps=STEP_SIZE_VALID )
d.remove(d[7] )<feature_engineering>
def get_results(model_used):
test_generator.reset()
pred=model_used.predict_generator(test_generator,
steps=STEP_SIZE_TEST+1,
verbose=1)
 predicted_class_indices=np.argmax(pred,axis=1)
 labels =(train_generator.class_indices)
 labels = dict(( v,k)for k,v in labels.items())
 predictions = [labels[k] for k in predicted_class_indices]
filenames=test_generator.filenames
results=pd.DataFrame({"id":[f[:-4] for f in filenames],
"label":[x[0] for x in pred]})
 return results
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
<feature_engineering>
results = get_results(VGG19_model )
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )<feature_engineering>
results.to_csv("results.csv",
index=False )
<groupby><EOS> 
!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
!unzip ngrok-stable-linux-amd64.zip
LOG_DIR = './logs'
get_ipython().system_raw(
 'tensorboard --logdir {} --host 0.0.0.0 --port 8080 &' 
.format(LOG_DIR)
)
 get_ipython().system_raw('./ngrok http 8080 &')
 ! curl -s http://localhost:4040/api/tunnels | python3 -c \
"