Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<choose_model_class>
print("Currently using Tensorflow version " + tf.__version__ )
data_augment = ImageDataGenerator(rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1 )<choose_model_class>
DEVICE = 'TPU'
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(32,(5,5), padding='same', activation='relu', input_shape=(28,28,1)) ,
tf.keras.layers.Conv2D(32,(5,5), padding='same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(64,(3,3), padding='same', activation='relu'),
tf.keras.layers.Conv2D(64,(3,3), padding='same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.summary()<choose_model_class>
SEED = 34
IMAGE_SIZE = [256, 256]
BATCH_SIZE = 32 * REPLICAS
EPOCHS = 15
FOLDS = 5
UPSAMPLE = False
TTA = 15
VERBOSE = 2
annealer = ReduceLROnPlateau(
 monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
GCS_PATH = KaggleDatasets().get_gcs_path(f'melanoma-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}')
 GCS_PATH2 = KaggleDatasets().get_gcs_path(f'malignant-v2-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}' )
optim = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model.compile(optimizer=optim, loss="categorical_crossentropy", metrics=['accuracy'])
 num_epochs = 30
batch_size = 86
history = model.fit_generator(
 data_augment.flow(train_data, train_labels, batch_size=batch_size),
steps_per_epoch=train_data.shape[0]//batch_size,
validation_data=(valid_data, valid_labels),
epochs=num_epochs,
callbacks=[annealer]
)<save_to_csv>
print(GCS_PATH)
 print(GCS_PATH2 )
predictions = model.predict(test_data)
 pred_list = []
for index, pred in enumerate(predictions):
pred_list.append({"ImageId": index+1, "Label": np.argmax(pred)})
 sub_df = pd.DataFrame(pred_list)
 sub_df.to_csv("submission.csv", index=False )<set_options>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name = True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, dim, course_drop = False, grid_mask = False, mat_aug = False, all_aug = False,
droprate =.7, dropct = 8, dropsize =.2):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if course_drop:
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 if(droprate!=0)&(dropct!=0)&(dropsize!=0):
img = dropout(img, DIM = dim, PROBABILITY = droprate, CT = dropct, SZ = dropsize)
 if mat_aug:
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = transform(img,DIM=dim)
 if grid_mask:
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 img = apply_grid_mask(img,(dim, dim, 3))
if all_aug:
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 img = apply_all_aug(img, DIM = dim)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
warnings.filterwarnings('ignore' )<set_options>
AUTO = tf.data.experimental.AUTOTUNE
train_files = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
test_files = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
NUM_TRAINING_IMAGES = int(count_data_items(train_files)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(train_files)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(test_files)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8<load_from_csv>
def get_dataset(files, course_drop = False, grid_mask = False, mat_aug = False, all_aug = False, shuffle = False,
repeat = False, labeled = True, return_image_names = True, batch_size = BATCH_SIZE, dim = IMAGE_SIZE[0]):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(2048)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(
 prepare_image(img, course_drop = course_drop, grid_mask = grid_mask ,mat_aug = mat_aug,
all_aug = all_aug, dim=dim,), imgname_or_label), num_parallel_calls = AUTO)
 ds = ds.batch(batch_size)
 ds = ds.prefetch(AUTO)
 return ds
data_raw = pd.read_csv('.. /input/train.csv')
 data_val = pd.read_csv('.. /input/test.csv')
 data1 = data_raw.copy(deep = True)
 data_cleaner = [data1, data_val]
print(data_raw.info())
 data_raw.sample(10 )<data_type_conversions>
classes = ['benign', 'malignant']
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
data1.drop(drop_column, axis=1, inplace = True)
 print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )<feature_engineering>
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return classes[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(classes[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
classes[correct_label] if not correct else ''), correct
def display_one_image(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):
images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else classes[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_image(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
for dataset in data_cleaner:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['Title'] = dataset['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 stat_min = 10
title_names =(data1['Title'].value_counts() < stat_min)
 data1['Title'] = data1['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(data1['Title'].value_counts())
 print("-"*10)
 data1.info()
data_val.info()
data1.sample(10 )<categorify>
training_dataset = get_dataset(train_files, labeled = True, course_drop = False, all_aug = False,
grid_mask = False, mat_aug = False, shuffle = True, repeat = True)
 training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )
label = LabelEncoder()
for dataset in data_cleaner:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['Title_Code'] = label.fit_transform(dataset['Title'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
data1_x = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
data1_x_calc = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code','SibSp', 'Parch', 'Age', 'Fare']
data1_xy = Target + data1_x
print('Original X Y: ', data1_xy, '
')
 data1_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
data1_xy_bin = Target + data1_x_bin
print('Bin X Y: ', data1_xy_bin, '
')
 data1_dummy = pd.get_dummies(data1[data1_x])
 data1_x_dummy = data1_dummy.columns.tolist()
data1_xy_dummy = Target + data1_x_dummy
print('Dummy X Y: ', data1_xy_dummy, '
')
 data1_dummy.head()<split>
test_dataset = get_dataset(test_files, labeled = False, course_drop = False, grid_mask = False,
mat_aug = False, all_aug = False, shuffle = True, repeat = False)
 test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data1[data1_x_calc], data1[Target], random_state = 0)
 train1_x_bin, test1_x_bin, train1_y_bin, test1_y_bin = model_selection.train_test_split(data1[data1_x_bin], data1[Target] , random_state = 0)
 train1_x_dummy, test1_x_dummy, train1_y_dummy, test1_y_dummy = model_selection.train_test_split(data1_dummy[data1_x_dummy], data1[Target], random_state = 0)
 print("Data1 Shape: {}".format(data1.shape))
print("Train1 Shape: {}".format(train1_x.shape))
print("Test1 Shape: {}".format(test1_x.shape))
train1_x_bin.head()<set_options>
display_batch_of_images(next(train_batch))
plt.figure(figsize=[16,12])
 plt.subplot(231)
 plt.boxplot(x=data1['Fare'], showmeans = True, meanline = True)
 plt.title('Fare Boxplot')
 plt.ylabel('Fare($)')
 plt.subplot(232)
 plt.boxplot(data1['Age'], showmeans = True, meanline = True)
 plt.title('Age Boxplot')
 plt.ylabel('Age(Years)')
 plt.subplot(233)
 plt.boxplot(data1['FamilySize'], showmeans = True, meanline = True)
 plt.title('Family Size Boxplot')
 plt.ylabel('Family Size(
 plt.subplot(234)
 plt.hist(x = [data1[data1['Survived']==1]['Fare'], data1[data1['Survived']==0]['Fare']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Fare Histogram by Survival')
 plt.xlabel('Fare($)')
 plt.ylabel('
plt.legend()
plt.subplot(235)
 plt.hist(x = [data1[data1['Survived']==1]['Age'], data1[data1['Survived']==0]['Age']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Age Histogram by Survival')
 plt.xlabel('Age(Years)')
 plt.ylabel('
plt.legend()
plt.subplot(236)
 plt.hist(x = [data1[data1['Survived']==1]['FamilySize'], data1[data1['Survived']==0]['FamilySize']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Family Size Histogram by Survival')
 plt.xlabel('Family Size(
 plt.ylabel('
plt.legend()<choose_model_class>
display_batch_of_images(next(test_batch))
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare 
<define_variables>
ROT_ = 180.0; SHR_ = 2.0
HZOOM_ = 8.0; WZOOM_ = 8.0
HSHIFT_ = 8.0; WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
for index, row in data1.iterrows() :
if random.random() >.5:
data1.set_value(index, 'Random_Predict', 1)
 else:
data1.set_value(index, 'Random_Predict', 0)
 data1['Random_Score'] = 0
data1.loc[(data1['Survived'] == data1['Random_Predict']), 'Random_Score'] = 1
print('Coin Flip Model Accuracy: {:.2f}%'.format(data1['Random_Score'].mean() *100))
print('Coin Flip Model Accuracy w/SciKit: {:.2f}%'.format(metrics.accuracy_score(data1['Survived'], data1['Random_Predict'])*100))<groupby>
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )<create_dataframe>
def dropout(image, DIM=256, PROBABILITY = 1, CT = 8, SZ = 0.2):
prob = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(prob==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* prob
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict))<train_on_grid>
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.4
}
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Training w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
<save_to_csv>
def transform2(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform2(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape = IMAGE_SIZE):
mask = GridMask(image_shape[0],
image_shape[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return tf.cast(image * tf.cast(mask, tf.float32), tf.float32 )
dot_data = tree.export_graphviz(dtree, out_file=None,
feature_names = data1_x_bin, class_names = True,
filled = True, rounded = True)
 graph = graphviz.Source(dot_data)
 graph<choose_model_class>
def apply_all_aug(img, no_grid = False, DIM = IMAGE_SIZE[0]):
if tf.random.uniform([],0,1)>.5:
if tf.random.uniform([],0,1)>.5:
img = transform(img)
 else:
img = dropout(img, PROBABILITY = 1)
 else:
if not no_grid:
if tf.random.uniform([],0,1)>.5:
img = apply_grid_mask(img)
 else:
img = img
else:
if tf.random.uniform([],0,1)>.5:
img = transform(img)
 else:
img = img
return tf.reshape(img,[DIM,DIM,3] )
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Training w/bin score mean: {:.2f}".format(vote_hard_cv['train_score'].mean() *100))
print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Training w/bin score mean: {:.2f}".format(vote_soft_cv['train_score'].mean() *100))
print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10 )<define_search_space>
STEPS_PER_EPOCH = count_data_items(train_files)// BATCH_SIZE
LR_START = 5e-6
LR_MAX = 5e-6 * 8
LR_MIN = 1e-5
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_DECAY =.8
def lr_schedule(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lr_schedule(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
grid_n_estimator = [10, 50, 100, 300]
grid_ratio = [.1,.25,.5,.75, 1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2, 4, 6, 8, 10, None]
grid_min_samples = [5, 10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
grid_param = [
[{
'n_estimators': grid_n_estimator,
'learning_rate': grid_learn,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'max_samples': grid_ratio,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'learning_rate': [.05],
'n_estimators': [300],
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'oob_score': [True],
'random_state': grid_seed
}],
[{
'max_iter_predict': grid_n_estimator,
'random_state': grid_seed
}],
[{
'fit_intercept': grid_bool,
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'random_state': grid_seed
}],
[{
'alpha': grid_ratio,
}],
[{}],
[{
'n_neighbors': [1,2,3,4,5,6,7],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']
}],
[{
'C': [1,2,3,4,5],
'gamma': grid_ratio,
'decision_function_shape': ['ovo', 'ovr'],
'probability': [True],
'random_state': grid_seed
}],
[{
'learning_rate': grid_learn,
'max_depth': [1,2,4,6,8,10],
'n_estimators': grid_n_estimator,
'seed': grid_seed
}]
]
start_total = time.perf_counter()
for clf, param in zip(vote_est, grid_param):
start = time.perf_counter()
best_search = model_selection.GridSearchCV(estimator = clf[1], param_grid = param, cv = cv_split, scoring = 'roc_auc')
 best_search.fit(data1[data1_x_bin], data1[Target])
 run = time.perf_counter() - start
best_param = best_search.best_params_
print('The best parameter for {} is {} with a runtime of {:.2f} seconds.'.format(clf[1].__class__.__name__, best_param, run))
clf[1].set_params(**best_param)
 run_total = time.perf_counter() - start_total
print('Total optimization time was {:.2f} minutes.'.format(run_total/60))
print('-'*10 )<train_on_grid>
!pip install -q efficientnet
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 data_val['Survived'] = mytree(data_val ).astype(int)
 submit = data_val[['PassengerId','Survived']]<save_to_csv>
def get_DenseNet201() :
with strategy.scope() :
dnet = DenseNet201(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 dnet.trainable = True
model = tf.keras.Sequential([
dnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_Xception() :
with strategy.scope() :
xception = Xception(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 xception.trainable = True
model = tf.keras.Sequential([
xception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_InceptionV3() :
with strategy.scope() :
inception = InceptionV3(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception.trainable = True
model = tf.keras.Sequential([
inception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_InceptionResNetV2() :
with strategy.scope() :
inception_res = InceptionResNetV2(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception_res.trainable = True
model = tf.keras.Sequential([
inception_res,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB4() :
with strategy.scope() :
efficient = efn.EfficientNetB4(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB5() :
with strategy.scope() :
efficient = efn.EfficientNetB5(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB6() :
with strategy.scope() :
efficient = efn.EfficientNetB6(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt, loss = loss,metrics = ['AUC'])
 return model
submit.to_csv("titanic_submission.csv", index = False )<import_modules>
histories = []
preds = np.zeros(( count_data_items(test_files),1))
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for f,(train_index,val_index)in enumerate(kfold.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
train_files = tf.io.gfile.glob([GCS_PATH+ '/train%.2i*.tfrec'%x for x in train_index])
 if UPSAMPLE:
train_files += tf.io.gfile.glob([GCS_PATH2 + '/train%.2i*.tfrec'%x for x in train_index])
 np.random.shuffle(train_files)
 val_files = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in val_index])
 test_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))
 train_ds = get_dataset(train_files,labeled = True, return_image_names = False, all_aug = True,
mat_aug = False, course_drop = False, repeat = True, shuffle = True)
 val_ds = get_dataset(val_files, mat_aug = False, course_drop = False, all_aug = False, grid_mask = False,
repeat = False, shuffle = False, labeled = True, return_image_names = False)
 sv = tf.keras.callbacks.ModelCheckpoint(
 f'fold-{f}.h5', monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min', save_freq = 'epoch')
 print('Training...'); print('')
 model = get_EfficientNetB4()
history = model.fit(train_ds, validation_data = val_ds, callbacks = [sv, lr_callback],
verbose = VERBOSE, steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS 
)
 histories.append(history)
 print('Loading best model...')
 model.load_weights(f'fold-{f}.h5')
 if TTA > 0:
ds_test = get_dataset(test_files,labeled = False,return_image_names = False,mat_aug = True,
course_drop = False, repeat = True,shuffle = False)
 ct_test = count_data_items(test_files); STEPS = TTA * ct_test/BATCH_SIZE
pred = model.predict(ds_test,steps = STEPS,verbose = VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order = 'F'),axis = 1)* 1 / FOLDS
else:
ds_test = get_dataset(test_files,labeled = False,return_image_names = False, mat_aug = False,
course_drop = False, repeat = False, shuffle = False)
 pred = model.predict(ds_test,verbose = VERBOSE)
 preds += pred * 1 / FOLDS
del model; z = gc.collect()
import numpy as np
import pandas as pd
import random 
<import_modules>
ds = get_dataset(test_files, course_drop = False, repeat = False, mat_aug = False,
dim = IMAGE_SIZE[0], labeled = False, return_image_names = True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 submission = pd.DataFrame(dict(image_name = image_names, target = preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index = False)
 submission.head(10 )
<import_modules><EOS> 
print(f"Predicted {len(submission[submission['target'] >.5])} images with melanoma" )