Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<split>
notebookstart = time.time()
train_data,val_data=train_test_split(train_df,test_size=0.15)
 train_data,val_data=train_data.reset_index(drop=True),val_data.reset_index(drop=True )<define_variables>
warnings.filterwarnings('ignore')
 warnings.simplefilter(action='ignore', category=DeprecationWarning)
 warnings.simplefilter(action='ignore', category=UserWarning )
class My_Generator(K.utils.Sequence):
def __init__(self, image_filenames, labels,
batch_size,path, is_train=True,
mix=False, augment=False):
self.image_filenames, self.labels = image_filenames, labels
self.batch_size = batch_size
self.is_train = is_train
self.is_augment = augment
self.path=path
if(self.is_train):
self.on_epoch_end()
self.is_mix = mix
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
return self.train_generate(batch_x, batch_y)
 return self.valid_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_train):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def mix_up(self, x, y):
lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def train_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread(os.path.join(self.path,sample))
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(SIZE, SIZE))
if(self.is_augment):
img = seq.augment_image(img)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 batch_y=to_categorical(batch_y,num_classes=5)
 if(self.is_mix):
batch_images, batch_y = self.mix_up(batch_images, batch_y)
 return batch_images, batch_y
def valid_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread(os.path.join(self.path,sample))
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(SIZE, SIZE))
batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 batch_y=to_categorical(batch_y,num_classes=5)
 return batch_images, batch_y<predict_on_test>
import platform
import sys
import importlib
import multiprocessing
import random
class QWKEvaluation(Callback):
def __init__(self, validation_data=() , batch_size=batch_size, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.batch_size = batch_size
self.valid_generator, self.y_val = validation_data
self.history = []
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_generator(generator=self.valid_generator,
steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)) ,
workers=1, use_multiprocessing=False,
verbose=1)
 def flatten(y):
return np.argmax(y, axis=1 ).reshape(-1)
 score = cohen_kappa_score(self.y_val,
flatten(y_pred),
labels=[0,1,2,3,4],
weights='quadratic')
 print("
epoch: %d - QWK_score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history):
print('saving checkpoint: ', score)
 self.model.save('.. /working/classifier_6.h5')
<choose_model_class>
random.seed(321)
 np.random.seed(321)
 pd.options.display.max_columns = 9999
train_generator=My_Generator(np.asarray(train_data['new_id_col']),np.asarray(train_data['diagnosis']),
batch_size=batch_size,path=train_img_path,is_train=True,mix=False,augment=True)
 val_generator=My_Generator(np.asarray(val_data['new_id_col']),np.asarray(val_data['diagnosis']),
batch_size=batch_size,path=train_img_path,is_train=False,mix=False,augment=False )<compute_test_metric>
%matplotlib inline
mpl.rc('figure', figsize=(15, 12))
plt.figure(figsize=(15, 12))
plt.rcParams['figure.facecolor'] = 'azure'
mpl.style.use('seaborn')
 plt.style.use('seaborn')
 set_matplotlib_formats('retina' )
qwk = QWKEvaluation(validation_data=(val_generator, np.asarray(val_data['diagnosis'])) ,
batch_size=batch_size, interval=1 )<choose_model_class>
sns.set(rc={'figure.figsize':(15, 12)})
 sns.set(context='notebook', style='darkgrid', font='sans-serif',
font_scale=1.1, rc={'figure.facecolor': 'azure',
'axes.facecolor': 'azure', 'grid.color': 'steelblue'})
 sns.color_palette(mcolors.TABLEAU_COLORS);
reg = l1(0.001 )<choose_model_class>
import missingno as msno
base_layer=K.applications.densenet.DenseNet169(input_shape=(224,224,3),include_top=False,
weights='.. /input/densenet-keras/DenseNet-BC-169-32-no-top.h5')
 for layer in base_layer.layers:
layer.trainable=True
lstm_=K.layers.Reshape([7*7,1664] )(base_layer.output)
 lstm_=K.layers.Bidirectional(K.layers.LSTM(832,recurrent_dropout=0.2))(lstm_)
 x=K.layers.Dense(1024,activation='relu',activity_regularizer=reg )(lstm_)
 x=K.layers.Dropout(0.3 )(x)
 out=K.layers.Dense(5,activation='softmax' )(x )<choose_model_class>
import scikitplot as skplt
model.compile(loss='categorical_crossentropy',optimizer=Adam(0.0001))<choose_model_class>
cross_val_predict, cross_val_score
StratifiedKFold, GridSearchCV
RobustScaler, MaxAbsScaler, Normalizer
classification_report
precision_recall_fscore_support
rp=ReduceLROnPlateau(monitor='val_loss',factor=0.2,min_lr=0.00000001,patience=3,verbose=1)
 mc=ModelCheckpoint('.. /working/classifier_6.h5',monitor='val_loss',mode='min',period=1,save_best_only=True )<train_model>
from sklearn.cluster import KMeans
from sklearn.ensemble import RandomForestClassifier
history=model.fit_generator(train_generator,epochs=30,steps_per_epoch=int(np.ceil(len(train_data)/batch_size)) ,
validation_data=val_generator,
validation_steps=int(np.ceil(len(val_data)/batch_size)) ,callbacks=[qwk,rp]
,class_weight=class_weights )<drop_column>
from scipy import stats
del train_generator,val_generator,train_data,val_data<load_pretrained>
import xgboost as xgb
from xgboost import XGBClassifier, plot_tree, plot_importance
model.load_weights('.. /working/classifier_6.h5')
 print('Weights Restored' )<load_from_csv>
import lightgbm as lgbm
from lightgbm import LGBMClassifier
test_df=pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 predictions=[]
for id_ in tqdm(test_df['id_code']):
img = cv2.imread(os.path.join(test_images_path,id_+'.png'))
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(SIZE, SIZE))
img=img/255.0
img=np.expand_dims(img,0)
 predictions.extend(model.predict(img))<save_to_csv>
import catboost
from catboost import CatBoostClassifier
predictions=np.argmax(predictions,1)
 sub=pd.DataFrame({'id_code':test_df['id_code'],'diagnosis':predictions})
 sub.to_csv('submission.csv',index=False )<define_variables>
import skopt
from skopt import BayesSearchCV
IMSIZE = 300
aptos2019_path = ".. /input/aptos2019-blindness-detection/"
aptos2019_train_path = ".. /input/aptos2019-blindness-detection/train_images/"
aptos2019_test_path = ".. /input/aptos2019-blindness-detection/test_images/"
aptos2019_extension = ".png"
aptos2015_path = ".. /input/aptos2015blindnessdetection/"
aptos2015_train_path = ".. /input/aptos2015blindnessdetection/train_images/"
aptos2015_test_path = ".. /input/aptos2015blindnessdetection/test_images/"
aptos2015_extension = ".jpg"
def plot_diag_hist(dataframe, title='NoTitle'):
f, ax = plt.subplots(figsize=(7, 4))
ax = sns.countplot(x="diagnosis", data=dataframe, palette="GnBu_d")
 sns.despine()
plt.title(title)
 plt.show()
trainbase_2019_df = pd.read_csv(aptos2019_path+"train.csv")
 test_2019_df = pd.read_csv(aptos2019_path+"test.csv")
 print("Shape of 2019 Training Data: {}".format(trainbase_2019_df.shape))
print("Shape of 2019 Test Data: {}
".format(test_2019_df.shape))
{}
".format(trainbase_2019_df.head()))
{}
".format(trainbase_2019_df['diagnosis'].value_counts() /trainbase_2019_df.shape[0]))
trainbase_2015_df = pd.read_csv(aptos2015_path+"train.csv")
 test_2015_df = pd.read_csv(aptos2015_path+"test.csv")
 print("Shape of 2015 Training Data: {}".format(trainbase_2015_df.shape))
print("Shape of 2015 Test Data: {}
".format(test_2015_df.shape))
trainbase_2015_df.rename(columns={"image": "id_code", "level": "diagnosis"}, inplace=True)
 test_2015_df.rename(columns={"image": "id_code", "level": "diagnosis"}, inplace=True)
 test_2015_df.drop(columns=["Usage"], inplace=True)
 plot_diag_hist(trainbase_2019_df, title="Predictions of 2019 Training Data")
 plot_diag_hist(trainbase_2015_df, title="Predictions of 2015 Training Data")
 full_path_train_2019 = lambda id_code : aptos2019_train_path+id_code+aptos2019_extension
full_path_test_2019 = lambda id_code : aptos2019_test_path+id_code+aptos2019_extension
full_path_train_2015 = lambda id_code : aptos2015_train_path+id_code+aptos2015_extension
full_path_test_2015 = lambda id_code : aptos2015_test_path+id_code+aptos2015_extension
trainbase_2019_df["path"] = trainbase_2019_df["id_code"].apply(full_path_train_2019)
 test_2019_df["path"] = test_2019_df["id_code"].apply(full_path_test_2019)
 trainbase_2015_df["path"] = trainbase_2015_df["id_code"].apply(full_path_train_2015)
 test_2015_df["path"] = test_2015_df["id_code"].apply(full_path_test_2015)
<split>
import imblearn
from imblearn.over_sampling import SMOTE, SMOTENC, RandomOverSampler
from imblearn.combine import SMOTEENN, SMOTETomek
train_raw_df, validation_df = train_test_split(trainbase_2019_df, test_size=0.2, random_state=42,
stratify=trainbase_2019_df[['diagnosis']])
 print("Shape of Training Data: {}".format(train_raw_df.shape))
print("Diagnosis Distribution in Training Set:
{}
".format(train_raw_df['diagnosis'].value_counts() /train_raw_df.shape[0]))
print("Shape of Validation Data: {}".format(validation_df.shape))
print("Diagnosis Distribution in Validation Set:
{}
".format(validation_df['diagnosis'].value_counts() /validation_df.shape[0]))
train_raw_2015_df, validation_2015_df = train_test_split(trainbase_2015_df, test_size=0.2, random_state=42,
stratify=trainbase_2015_df[['diagnosis']])
 print("Shape of additional 2015 Training Data: {}".format(train_raw_2015_df.shape))
print("Diagnosis Distribution in 2015 Training Set:
{}
".format(train_raw_2015_df['diagnosis'].value_counts() /train_raw_2015_df.shape[0]))
print("Shape of additional 2015 Validation Data: {}".format(validation_2015_df.shape))
print("Diagnosis Distribution in 2015 Validation Set:
{}
".format(validation_2015_df['diagnosis'].value_counts() /validation_2015_df.shape[0]))
train_df = train_raw_df
train_2015_df = train_raw_2015_df
print("Shape of Oversampled Training Data: {}".format(train_df.shape))
print("Shape of Merged Validation Data: {}".format(validation_df.shape))
print("Appearance of Oversampled Training Data:
{}
".format(train_df.head()))
print("Diagnosis Distribution in Oversampled Training Set:
{}
".format(train_df['diagnosis'].value_counts() /train_df.shape[0]))
plot_diag_hist(train_df, title="Predictions of Stratified and Oversampled Training Data")
 train_df = train_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 validation_df = validation_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 train_2015_df = train_2015_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 validation_2015_df = validation_2015_df.sample(frac=1, random_state=42 ).reset_index(drop=True )<normalization>
print('Operating system version........ ', platform.platform())
 print('Python version is...............%s.%s.%s' % sys.version_info[:3])
 print('scikit-learn version is.........', sklearn.__version__)
 print('pandas version is...............', pd.__version__)
 print('numpy version is................ ', np.__version__)
 print('matplotlib version is...........', mpl.__version__)
 print('seaborn version is.............. ', sns.__version__)
 print('scikit-plot version is.......... ', skplt.__version__)
 print('missingno version is............ ', msno.__version__)
 print('xgboost version is.............. ', xgb.__version__)
 print('catboost version is.............', catboost.__version__)
 print('lightgbm version is.............', lgbm.__version__)
 print('scikit-optimize version is...... ', skopt.__version__)
 print('imblearn version is.............', imblearn.__version__ )
imagepath = train_df.loc[4,"path"]
img = load_image_preprocess(imagepath, augmentation=False)
 plt.imshow(img)
 print(img.shape)
 print(( img[np.newaxis] ).shape)
 display_samples_preprocess_steps(train_df)
 img1 = augment_aptos(aug_rotate,img)
 img2 = augment_aptos(aug_rotate,img)
 show_Nimages([img,img1,img2],scale=2)
 img1 = augment_aptos(aug_flip,img)
 img2 = augment_aptos(aug_flip,img)
 img3 = augment_aptos(aug_flip,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_bright,img)
 img2 = augment_aptos(aug_bright,img)
 img3 = augment_aptos(aug_bright,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_crop,img)
 img2 = augment_aptos(aug_crop,img)
 img3 = augment_aptos(aug_crop,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_hole,img)
 img2 = augment_aptos(aug_hole,img)
 img3 = augment_aptos(aug_hole,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_flare,img)
 img2 = augment_aptos(aug_flare,img)
 img3 = augment_aptos(aug_flare,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(full_augmentation,img)
 img2 = augment_aptos(full_augmentation,img)
 img3 = augment_aptos(full_augmentation,img)
 show_Nimages([img3,img1,img2],scale=2 )<prepare_x_and_y>
def getDatasetInformation(csv_filepath, is_corr_required=True):

 dataset_tmp = pd.read_csv(csv_filepath)
 dataset_tmp_shape = pd.DataFrame(list(dataset_tmp.shape),
index=['No of Rows', 'No of Columns'], columns=['Total'])
 dataset_tmp_shape = dataset_tmp_shape.reset_index()
dataset_tmp_dtypes = dataset_tmp.dtypes.reset_index()
dataset_tmp_dtypes.columns = ['Column Names', 'Column Data Types']
dataset_tmp_desc = pd.DataFrame(dataset_tmp.describe())
 dataset_tmp_desc = dataset_tmp_desc.transpose()
dataset_tmp_unique = dataset_tmp.nunique().reset_index()
dataset_tmp_unique.columns = ['Column Name', 'Unique Value(s)Count'
]
dataset_tmp_missing = dataset_tmp.isnull().sum(axis=0 ).reset_index()
dataset_tmp_missing.columns = ['Column Names',
'NULL value count per Column']
dataset_tmp_missing = \
dataset_tmp_missing.sort_values(by='NULL value count per Column'
, ascending=False)
 if is_corr_required:
dataset_tmp_corr = dataset_tmp.corr(method='spearman')
 else:
dataset_tmp_corr = pd.DataFrame()
return [
dataset_tmp,
dataset_tmp_shape,
dataset_tmp_dtypes,
dataset_tmp_desc,
dataset_tmp_unique,
dataset_tmp_missing,
dataset_tmp_corr,
]
IS_REGRESSION = False
def extract_target_vector_multi(dataframe_X):
y_targets = pd.get_dummies(dataframe_X['diagnosis'] ).values
y_targets_multi = np.empty(y_targets.shape, dtype=y_targets.dtype)
 y_targets_multi[:, 4] = y_targets[:, 4]
for i in range(3, -1, -1):
y_targets_multi[:, i] = np.logical_or(y_targets[:, i], y_targets_multi[:, i+1])
 return y_targets_multi
def extract_target_vector_regression(dataframe_X):
y_targets = dataframe_X['diagnosis'].values
return y_targets
def result_from_prediction_regression(y_pred):
return(np.clip(np.round(y_pred),0,4)).astype("uint" ).flatten()
def result_from_prediction_multi(y_pred):
return(y_pred > 0.5 ).astype(int ).sum(axis=1)- 1
if(IS_REGRESSION):
y_train_multi = extract_target_vector_regression(train_df)
 y_validate_multi = extract_target_vector_regression(validation_df)
 y_trainbase_2019_multi = extract_target_vector_regression(trainbase_2019_df)
 y_trainbase_2015_multi = extract_target_vector_regression(trainbase_2015_df)
 y_test_2015_multi = extract_target_vector_regression(test_2015_df)
 result_from_prediction = result_from_prediction_regression
else:
y_train_multi = extract_target_vector_multi(train_df)
 y_validate_multi = extract_target_vector_multi(validation_df)
 y_trainbase_2019_multi = extract_target_vector_multi(trainbase_2019_df)
 y_trainbase_2015_multi = extract_target_vector_multi(trainbase_2015_df)
 y_test_2015_multi = extract_target_vector_multi(test_2015_df)
 result_from_prediction = result_from_prediction_multi
print("Training data:
Xshape: {}
yshape: {}
".format(train_df.shape,y_train_multi.shape))
print("Validation data:
Xshape: {}
yshape: {}
".format(validation_df.shape,y_validate_multi.shape))
print("Test data:
Xshape: {}
".format(test_2019_df.shape))
print("Training data multilabel diagnosis vector:", y_train_multi.sum(axis=0))
print("Validation data multilabel diagnosis vector:", y_validate_multi.sum(axis=0))
print("Training data 2015 multilabel diagnosis vector:", y_trainbase_2015_multi.sum(axis=0))
print("Training data 2015 multilabel diagnosis vector:", y_test_2015_multi.sum(axis=0))<define_variables>
def createFeatureEngineeredColumns(dataset):
dataset_tmp = pd.DataFrame()
dataset_tmp['CountOfZeroValues'] =(dataset == 0 ).sum(axis=1)
 dataset_tmp['CountOfNonZeroValues'] =(dataset != 0 ).sum(axis=1)
 weight =(( dataset != 0 ).sum() / len(dataset)).values
dataset_tmp['WeightedCount'] =(dataset * weight ).sum(axis=1)
 dataset_tmp['SumOfValues'] = dataset.sum(axis=1)
 dataset_tmp['VarianceOfValues'] = dataset.var(axis=1)
 dataset_tmp['MedianOfValues'] = dataset.median(axis=1)
 dataset_tmp['MeanOfValues'] = dataset.mean(axis=1)
 dataset_tmp['StandardDeviationOfValues'] = dataset.std(axis=1)
 dataset_tmp['SkewOfValues'] = dataset.skew(axis=1)
 dataset_tmp['KurtosisOfValues'] = dataset.kurtosis(axis=1)
 dataset_tmp['MaxOfValues'] = dataset.max(axis=1)
 dataset_tmp['MinOfValues'] = dataset.min(axis=1)
 dataset_tmp['DiffOfMinMaxOfValues'] = \
np.subtract(dataset_tmp['MaxOfValues'],
dataset_tmp['MinOfValues'])
 dataset_tmp['QuantilePointFiveOfValues'] = dataset[dataset
> 0].quantile(0.5, axis=1)
 dataset_tmp['MovingAverage'] = dataset.apply(lambda x: \
np.ma.average(x), axis=1)
 percentileList = [ 1, 2, 5, 10, 25, 50, 60, 75, 80, 85, 95, 99, ]
for i in percentileList:
dataset_tmp['Percentile' + str(i)] = dataset.apply(lambda x: \
np.percentile(x, i), axis=1)
 for column in dataset.columns: 
(hist, bin_edges)= np.histogram(dataset[column], bins=1000,
density=True)
 dataset_tmp[column + '_histval'] = \
[hist[np.searchsorted(bin_edges, ele)- 1] for ele in
dataset[column]]
for column in dataset.columns:
dataset_tmp[column + '_log'] = np.log(dataset[column])
 for column in dataset.columns:
dataset_tmp[column + '_square'] = np.square(dataset[column])
 dataset_tmp = dataset_tmp.fillna(0)
 dataset = pd.concat([dataset, dataset_tmp], axis=1)
 return dataset
BATCH_SIZE = 32
class My_Image_Generator(Sequence):
def __init__(self, image_filenames, labels, batch_size, shuffle=True, augment=False, train=False):
self.image_filenames, self.labels = image_filenames, labels
self.is_train = train
self.batch_size = batch_size
self.is_shuffle = shuffle
self.is_augment = augment
if(self.is_shuffle):
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
else:
batch_y = np.zeros(self.batch_size)
 return self.batch_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_shuffle):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def batch_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
if(self.is_augment):
img = load_image_preprocess(sample, augmentation=True)
 else:
img = load_image_preprocess(sample, augmentation=False)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
train_data_generator_aug = My_Image_Generator(train_df["path"], labels=y_train_multi, shuffle=True, train=True,
batch_size=BATCH_SIZE, augment=True)
 train_data_generator_no_aug = My_Image_Generator(train_df["path"], labels=y_train_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False)
 valid_data_generator_aug = My_Image_Generator(validation_df["path"], y_validate_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=True)
 valid_data_generator_no_aug = My_Image_Generator(validation_df["path"], y_validate_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False)
 trainbase_2019_data_generator_aug = My_Image_Generator(trainbase_2019_df["path"], labels=y_trainbase_2019_multi,
shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)
 trainbase_2019_data_generator_no_aug = My_Image_Generator(trainbase_2019_df["path"], labels=y_trainbase_2019_multi,
shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)
 trainbase_2015_data_generator_aug = My_Image_Generator(trainbase_2015_df["path"], labels=y_trainbase_2015_multi,
shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)
 trainbase_2015_data_generator_no_aug = My_Image_Generator(trainbase_2015_df["path"], labels=y_trainbase_2015_multi,
shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)
 test_data_generator_aug = My_Image_Generator(test_2019_df["path"], labels=None, shuffle=False, train=False,
batch_size=BATCH_SIZE, augment=True)
 test_data_generator_no_aug = My_Image_Generator(test_2019_df["path"], labels=None, shuffle=False, train=False,
batch_size=BATCH_SIZE, augment=False)
 test_data_2015_generator_no_aug = My_Image_Generator(test_2015_df["path"], labels=y_test_2015_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False )<train_model>
def getZeroStdColumns(dataset):
columnsWithZeroStd = dataset.columns[dataset.std() == 0].tolist()
return columnsWithZeroStd
class QWK_Metrics(Callback):
def __init__(self, model, validation_generator_X, validation_y):
self.model_to_save = model
self.valid_generator = validation_generator_X
self.valid_y = validation_y
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
y_val_pred = model.predict_generator(generator=self.valid_generator,
steps=-(self.valid_y.shape[0] // -BATCH_SIZE),
verbose=1)
 y_val_pred = result_from_prediction(y_val_pred)
 y_val_true = result_from_prediction(self.valid_y)
 _val_kappa = cohen_kappa_score(y_val_pred, y_val_true, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model_to_save.save('model_kappa.h5')
 return
reduceLR = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=5,
min_lr=1e-6,
verbose=1,
mode='min' 
)
 earlyStopping = EarlyStopping(
 monitor='val_loss',
patience=10,
verbose=1,
mode='min',
restore_best_weights=True 
)
<set_options>
def getUniqueValueColumns(dataset, valueToCheck=0):
columnsWithUniqueValue = dataset.columns[dataset.nunique()
== valueToCheck].tolist()
return columnsWithUniqueValue
def get_available_gpus() :
local_device_protos = device_lib.list_local_devices()
return [x.name for x in local_device_protos if x.device_type == "GPU"]
num_gpu = len(get_available_gpus())
 print("Number of available GPUs: {}".format(num_gpu))<choose_model_class>
def convertIntFloatToInt(dictObj):
for(k, v)in dictObj.items() :
if float('Inf')== v:
pass
elif int(v)== v and isinstance(v, float):
dictObj[k] = int(v)
 return dictObj
used_init = "lecun_uniform"
reg_value_kernel = 5e-5
reg_value_bias = 5e-5
def build_model_plainCNN() :
model = Sequential()
model.add(Conv2D(32,(5, 5), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(256,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(512,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(512, activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(Dropout(0.5))
model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))
return model
def build_model_flat() :
model = Sequential()
model.add(Conv2D(32,(9, 9), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Conv2D(64,(3, 3), activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Conv2D(128,(3, 3), activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu', kernel_initializer=used_init))
model.add(Dropout(0.2))
model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))
return model
def build_model_densenet121() :
densenet = DenseNet121(weights='.. /input/densenetpretrained/DenseNet-BC-121-32-no-top.h5', include_top=False,
input_shape=(IMSIZE,IMSIZE,3))
ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(densenet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB0() :
effnet = efn.EfficientNetB0(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnetpretrained/efficientnet-b0_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB1() :
effnet = efn.EfficientNetB1(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnetpretrained/efficientnet-b1_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB2() :
effnet = efn.EfficientNetB2(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/efficientnet-b2_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB3() :
effnet = efn.EfficientNetB3(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/pretrainedweights/efficientnet-b3_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 if(IS_REGRESSION):
ly_dense_2 = layers.Dense(1, activation='linear')
 else:
ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB4() :
effnet = efn.EfficientNetB4(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/efficientnet-b4_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
if(num_gpu == 0):
base_model = build_model_efficientnetB3()
model = base_model
print("Training on single CPU...no GPU avaliable")
 elif(num_gpu == 1):
base_model = build_model_efficientnetB3()
model = base_model
print("Training on single GPU...")
 else:
with tf.device('/cpu:0'):
base_model = build_model_efficientnetB3()
try:
model = multi_gpu_model(base_model, gpus=4)
 print("Training on multiple GPUs.. ")
 except ValueError:
model = base_model
print("Training on single GPU or CPU...error when compiling multi_gpu_model...")
 if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-4), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()<load_pretrained>
(
dataset_sctp_train,
df_train_shape,
df_train_dtypes,
df_train_describe,
df_train_unique,
df_train_missing,
df_train_corr, 
)= getDatasetInformation('.. /input/train.csv', False)
(
 dataset_sctp_test,
df_test_shape,
df_test_dtypes,
df_test_describe,
df_test_unique,
df_test_missing,
df_test_corr, 
)= getDatasetInformation('.. /input/test.csv', False )
for layer in model.layers:
layer.trainable = True
for layer in model.layers[:-5]:
layer.trainable = False
if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=1e-4), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-4), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()
kappa_metrics_1 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi)
 history_1 = model.fit_generator(
 generator=trainbase_2015_data_generator_aug,
steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=trainbase_2019_data_generator_no_aug,
validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_1, reduceLR, earlyStopping] 
)
 for layer in model.layers:
layer.trainable = True
if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()
kappa_metrics_2 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi)
 history_2 = model.fit_generator(
 generator=trainbase_2015_data_generator_aug,
steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=trainbase_2019_data_generator_no_aug,
validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_2, reduceLR, earlyStopping] 
)
 base_model.save('model_phase_1.h5')
<train_model>
msno.matrix(dataset_sctp_test, color=(33 / 255, 102 / 255, 172 / 255)) ;
kappa_metrics_3 = QWK_Metrics(base_model, valid_data_generator_no_aug, y_validate_multi)
 history_3 = model.fit_generator(
 generator=train_data_generator_aug,
steps_per_epoch=-(y_train_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=valid_data_generator_no_aug,
validation_steps=-(y_validate_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_3, reduceLR, earlyStopping] 
)
 base_model.save('model_phase_2.h5' )<find_best_params>
del(df_train_shape, df_train_dtypes, df_train_describe, df_train_unique, df_train_missing, df_train_corr)
 del(df_test_shape, df_test_dtypes, df_test_describe, df_test_unique, df_test_missing, df_test_corr )
kappa_metrics = kappa_metrics_3
history_df = pd.DataFrame(history_3.history)
 print("Max Kappa: {}".format(max(kappa_metrics.val_kappas)))
 best_epoch = np.argmax(kappa_metrics.val_kappas)+1
print("Max Kappa at epoch {}.".format(best_epoch))
print("Epoch {}: Acc: {:.4f}, ValAcc: {:.4f}, Loss: {:.4f}, ValLoss: {:.4f}".format(best_epoch,
history_df["acc"][best_epoch-1],
history_df["val_acc"][best_epoch-1],
history_df["loss"][best_epoch-1],
history_df["val_loss"][best_epoch-1] 
))<load_pretrained>
dataset_sctp_train.target.value_counts()
base_model.load_weights('model_kappa.h5')
<predict_on_test>
y = dataset_sctp_train['target']
X = dataset_sctp_train.drop(['ID_code', 'target'], axis=1)
 X.astype('float32')
 dataset_sctp_test_ID_code = dataset_sctp_test['ID_code']
Z = dataset_sctp_test.drop(['ID_code'], axis=1)
 Z.astype('float32')
(X.shape, y.shape, Z.shape )
tta_cycles = 5
y_test_tta = model.predict_generator(generator=test_data_generator_no_aug,
steps=-(test_2019_df.shape[0] // -BATCH_SIZE),
verbose=1)
 y_test_tta = result_from_prediction(y_test_tta)
 test_2019_df['diagnosis'] = y_test_tta
test_2019_df.drop(["path"], inplace=True, axis=1)
 test_2019_df.to_csv("submission.csv",index=False)
 print("Submission complete!")
<define_variables>
columnsWithZeroStdToRemove = getZeroStdColumns(X)
 print(f'Columns with Zero STD to drop from Train and Test dataset(s)are {columnsWithZeroStdToRemove}.')
 X.drop(columnsWithZeroStdToRemove, axis=1, inplace=True)
 Z.drop(columnsWithZeroStdToRemove, axis=1, inplace=True)
(X.shape, Z.shape )
package_path = '.. /input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'
sys.path.append(package_path )<import_modules>
n_cpus_avaliable = multiprocessing.cpu_count()
print(f'We've got {n_cpus_avaliable} cpus to work with.' )
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import torch.nn.functional as F
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
from sklearn.metrics import confusion_matrix
from torch.utils.data import random_split, Dataset
from torch.utils.data.sampler import SubsetRandomSampler
import os
from PIL import Image
import glob
import cv2
import gc
from tqdm import tqdm
import seaborn as sns
from sklearn.metrics import confusion_matrix, cohen_kappa_score
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm
from efficientnet_pytorch import EfficientNet<load_from_csv>
def augment(x, y, t=2): 
(xs, xn)=([], [])
 for i in range(t):
mask = y > 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:, c] = x1[ids][:, c]
xs.append(x1)
 for i in range(t // 2):
mask = y == 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:, c] = x1[ids][:, c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x, xs, xn])
 y = np.concatenate([y, ys, yn])
 return(x, y )
class APTOSDataset(Dataset):

 def __init__(self, csv_file, filetype, transform=None):
self.eye_frame = pd.read_csv(csv_file)
 self.filetype = filetype
self.transform = transform
def __len__(self):
return len(self.eye_frame)
 def __getitem__(self, idx):
if self.filetype == 'train':
img_name = os.path.join('.. /input/aptos2019-blindness-detection/train_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image,self.eye_frame.diagnosis[idx]
else:
img_name = os.path.join('.. /input/aptos2019-blindness-detection/test_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image, self.eye_frame.loc[idx,'id_code'] 
<load_from_csv>
lgbmparams = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"tree_learner": "serial",
'n_jobs': n_cpus_avaliable,
}
n_splits = 7
skf = StratifiedKFold(n_splits=n_splits, shuffle=True)
 sctp_predictions_lgbm_pp = np.zeros(len(Z))
for(index,(train_indices, val_indices)) in enumerate(skf.split(X, y)) :
print('Training on fold ' + str(index + 1)+ '/' + str(n_splits)+ '...')
(xtrain, xval)=(X.iloc[train_indices], X.iloc[val_indices])
(ytrain, yval)=(y.iloc[train_indices], y.iloc[val_indices])
(xtrain_aug, ytrain_aug)= augment(xtrain.values, ytrain.values,
t=2)
 xtrain_aug = pd.DataFrame(xtrain_aug, columns=X.columns)
 ytrain_aug = [int(value)for value in ytrain_aug]
ytrain_aug = pd.Series(ytrain_aug)
 train_set_lgbm = lgbm.Dataset(data=xtrain_aug, label=ytrain_aug)
 val_set_lgbm = lgbm.Dataset(data=xval, label=yval)
 watchlist = [train_set_lgbm, val_set_lgbm]
evaluation_results_lgbm = {}
lgbm_twoclass_model = lgbm.train(
 lgbmparams,
train_set=train_set_lgbm,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=1000,
evals_result=evaluation_results_lgbm,
verbose_eval=1000, 
)
 sctp_predictions_lgbm_pp += lgbm_twoclass_model.predict(Z,
num_iteration=lgbm_twoclass_model.best_iteration)\
/ skf.n_splits
class KFoldDataset(Dataset):

 def __init__(self, csv_file, indices, filetype, transform=None):
self.eye_frame = pd.read_csv(csv_file ).iloc[indices].reset_index(drop=True)
 self.filetype = filetype
self.transform = transform
def __len__(self):
return len(self.eye_frame)
 def __getitem__(self, idx):
if self.filetype == 'train':
img_name = os.path.join('.. /input/aptos2019-blindness-detection/train_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image,self.eye_frame.diagnosis[idx]
else:
img_name = os.path.join('.. /input/aptos2019-blindness-detection/test_images',
self.eye_frame.loc[idx,'id_code'] + '.png')
 image = Image.open(img_name)
 if self.transform:
image = self.transform(image)
 else:
image = transforms.ToTensor()(image)
 return image, self.eye_frame.loc[idx,'id_code']<load_from_csv>
sctp_predictions_lgbm = lgbm_twoclass_model.predict(Z)
 sctp_predictions_lgbm = [round(value)for value in sctp_predictions_lgbm]
sctp_predictions_lgbm = list(map(int, sctp_predictions_lgbm))
sctp_predictions_lgbm[:20]
<load_pretrained>
dataset_submission = pd.DataFrame()
dataset_submission['ID_code'] = dataset_sctp_test_ID_code
dataset_submission['target'] = sctp_predictions_lgbm
dataset_submission.to_csv('lgbm_twoclass_model_submission.csv', index=False )
<load_from_csv>
dataset_submission = pd.DataFrame()
dataset_submission['ID_code'] = dataset_sctp_test_ID_code
dataset_submission['target'] = sctp_predictions_lgbm_pp
dataset_submission.to_csv('lgbm_pp_twoclass_model_submission.csv', index=False )
<load_from_csv><EOS> 
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))