Kaggle competition: SIIM-ISIC Melanoma Classification
data['SexCode'] = LabelEncoder().fit_transform(data['Sex'])
 data['TitleCode'] = LabelEncoder().fit_transform(data['Title'])
 data['EmbarkedCode'] = LabelEncoder().fit_transform(data['Embarked'] )<data_type_conversions>
!pip install -q efficientnet
cols = [
'Pclass',
'Sex',
'FamilySize',
'SibSp',
'Parch',
'IsAlone',
'AgeBand',
'Fare',
'Title',
'DeckCode',
'RoomBand',
]
X_train = data[:train_size][cols]
Y_train = data[:train_size]['Survived'].astype(int)
 X_test = data[train_size:][cols]
print(X_train.shape, Y_train.shape, X_test.shape)
 X_train.head()<categorify>
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sample = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 train.head()
one_hot_features = [
'Sex',
'AgeBand',
'Title',
'DeckCode',
'RoomBand',
]
X_train = pd.get_dummies(X_train, columns = one_hot_features)
 X_test = pd.get_dummies(X_test, columns = one_hot_features)
 print(X_train.shape, Y_train.shape, X_test.shape )<choose_model_class>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", strategy.num_replicas_in_sync)
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 logreg.score(X_train, Y_train )<compute_train_metric>
SEED = 42
FOLDS=3
EFF_NETS = [6]*FOLDS
BATCH_SIZES = [bs * strategy.num_replicas_in_sync for bs in [32]*FOLDS]
IMG_SIZES = [512]*FOLDS
EPOCHS = [10]*FOLDS
DROPOUT = 0.25
LR = 0.00004
WARMUP = 5
CLASS_WEIGHT = {0: train['benign_malignant'].value_counts().malignant/len(train),
1: train['benign_malignant'].value_counts().benign/len(train)}
WGTS = [1/FOLDS]*FOLDS
LABEL_SMOOTHING = 0.05
TTA = 15
INC2018 = [1]*FOLDS
np.random.seed(SEED)
 tf.random.set_seed(SEED )
svc = SVC(C=9)
 svc.fit(X_train, Y_train)
 print(svc.score(X_train, Y_train))
scores = model_selection.cross_val_score(svc, X_train, Y_train, cv=5, scoring='accuracy')
 print(scores)
 print("Kfold on SVC: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std()))<find_best_model_class>
DATASET = {512: '512x512-melanoma-tfrecords-70k-images',
384: 'melanoma-384x384',
192: 'melanoma-192x192'}
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path(DATASET[IMG_SIZES[0]])
 GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
train_filenames = tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec' )
knn = KNeighborsClassifier(n_neighbors=9)
 knn.fit(X_train, Y_train)
 print(knn.score(X_train, Y_train))
scores = model_selection.cross_val_score(knn, X_train, Y_train, cv=5, scoring='accuracy')
 print(scores)
 print("Kfold on KNeighborsClassifier: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std()))<predict_on_test>
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.000003 * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.3
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 gaussian.score(X_train, Y_train )<compute_train_metric>
lr_start = 0.000005
lr_max = 0.000003 * BATCH_SIZES[0]
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.3
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
rng = [i for i in range(8 if EPOCHS[0]<8 else EPOCHS[0])]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
perceptron = Perceptron(max_iter=6)
 perceptron.fit(X_train, Y_train)
 perceptron.score(X_train, Y_train )<choose_model_class>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=512):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 decision_tree.score(X_train, Y_train )<train_model>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*IMG_SIZES[0:2], 3])
 return image
def read_tfrecord(example, labeled, return_imgname=False):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
if return_imgname:
return image, idnum
return image, 0
def load_dataset(filenames, labeled=True, ordered=False, return_imgname=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled,
return_imgname=return_imgname), num_parallel_calls=AUTO)
 , or(image, id)pairs if labeled=False
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def data_augment(image, label=None, seed=SEED):
image = transform(image, IMG_SIZES[0])
 image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 image = tf.image.random_brightness(image, max_delta=0.2)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 if label is None:
return image
else:
return image, label
def plot_transform(num_images):
fig, ax = plt.subplots(nrows=3, ncols=num_images, figsize=(12,5))
x =(load_dataset(train_filenames, labeled=True)
.shuffle(SEED)
.batch(BATCH_SIZES[0],drop_remainder=True)
.prefetch(AUTO)
.unbatch().take(5))
images = []
imgs=[]
for r in range(3):
image,_ = iter(x ).next()
images.append(image)
 for i in range(0,num_images):
image = data_augment(image=images[r])
 imgs.append(image)
 for img in range(len(imgs)) :
ax[img//num_images][img%num_images].imshow(imgs[img])
 ax[img//num_images][img%num_images].axis('off' )
sgd = SGDClassifier(max_iter=3)
 sgd.fit(X_train, Y_train)
 sgd.score(X_train, Y_train )<compute_train_metric>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 eg_ds =(load_dataset(train_filenames, labeled=True)
.batch(BATCH_SIZES[0],drop_remainder=True)
.prefetch(AUTO)
.unbatch().take(10*6))
show_dataset(64, 10, 6, eg_ds )
linear_svc = LinearSVC(C=0.07)
 linear_svc.fit(X_train, Y_train)
 print(linear_svc.score(X_train, Y_train))
scores = model_selection.cross_val_score(linear_svc, X_train, Y_train, cv=5, scoring='accuracy')
 print(scores)
 print("Kfold on LinearSVC: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std()))<find_best_model_class>
plot_transform(7 )
rf_params = {
'n_estimators': 700,
'max_depth': 5,
'min_samples_split': 10,
'min_samples_leaf': 1,
'max_features':'auto',
'oob_score':True,
}
random_forest = RandomForestClassifier(**rf_params)
 random_forest.fit(X_train, Y_train)
 print(random_forest.score(X_train, Y_train))
scores = model_selection.cross_val_score(random_forest, X_train, Y_train, cv=5, scoring='accuracy')
 print(scores)
 print("Kfold on RandomForestClassifier: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std()))<train_on_grid>
VERBOSE = 2
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(test_filenames),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 np.random.seed(SEED)
 np.random.shuffle(files_train); print('
train_dataset =(load_dataset(files_train, labeled=True)
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZES[fold],drop_remainder=True)
.prefetch(AUTO))
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB6(input_shape=(IMG_SIZES[fold],IMG_SIZES[fold], 3),
weights='imagenet',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=[tf.keras.metrics.AUC(name='auc')])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]))
print('Training...')
 history = model.fit(
 train_dataset,
epochs=EPOCHS[fold],
callbacks=[sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)// BATCH_SIZES[fold],
validation_data=load_dataset(files_valid, labeled=True)
.cache() 
.batch(BATCH_SIZES[fold])
.prefetch(AUTO),
verbose=VERBOSE)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid =(load_dataset(files_valid, labeled=True, ordered=True)
.cache() 
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
ct_valid = count_data_items(files_valid)
 STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid =(load_dataset(files_valid, labeled=True, ordered=True)
.cache() 
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds =(load_dataset(files_valid, labeled=False, return_imgname=True, ordered=True)
.cache() 
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test =(load_dataset(files_test, labeled=False, ordered=True)
.repeat() 
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZES[fold]*4)
.prefetch(AUTO))
ct_test = count_data_items(files_test)
 STEPS = TTA * ct_test/BATCH_SIZES[fold]/4
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',
label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',
label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.text(x-0.03*xdist,y-0.13*ydist,'max auc
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',
label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',
label='Val Loss',color='
x = np.argmin(history.history['val_loss'])
 y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)
 plt.ylabel('Loss',size=14)
 plt.xticks(ticks=list(range(EPOCHS[fold])) ,labels=list(range(1, EPOCHS[fold]+1)))
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
<choose_model_class>
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()
xg_boost = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=0.65, gamma=2, learning_rate=0.3, max_delta_step=1,
max_depth=4, min_child_weight=2, missing=None, n_estimators=280,
n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=True, subsample=1 )<find_best_model_class>
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(FOLDS):
fpr[i], tpr[i], _ = roc_curve(df_oof[df_oof.fold==i].target, df_oof[df_oof.fold==i].pred)
 roc_auc[i] = auc(fpr[i], tpr[i])
 fpr["mean"], tpr["mean"], _ = roc_curve(df_oof.target, df_oof.pred)
 roc_auc['mean'] = auc(fpr['mean'], tpr['mean'])
 precision = dict()
recall = dict()
average_precision = dict()
for i in range(FOLDS):
precision[i], recall[i], _ = precision_recall_curve(df_oof[df_oof.fold==i].target,
df_oof[df_oof.fold==i].pred)
 average_precision[i] = average_precision_score(df_oof[df_oof.fold==i].target,
df_oof[df_oof.fold==i].pred)
 precision["mean"], recall["mean"], _ = precision_recall_curve(df_oof.target, df_oof.pred)
 average_precision['mean'] = average_precision_score(df_oof.target, df_oof.pred)
 fig = plt.figure(figsize=(18,6))
ax1 = fig.add_subplot(121)
 ax1.plot([0, 1], [0, 1], linestyle='--', lw=4, color='r',
label='Chance', alpha=.8)
 colors = ['
'
for i in range(FOLDS):
ax1.plot(fpr[i], tpr[i], color=colors[i],
label=r'ROC Fold %i(AUC = %0.2f)' %(i+1,roc_auc[i]),
lw=3, alpha=.5)
 ax1.plot(fpr['mean'], tpr['mean'], color='b',
label=r'Mean ROC(AUC = %0.2f)' %(roc_auc['mean']),
lw=4, alpha=.8)
 ax1.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])
 plt.title("Receiver Operating Characteristic Curve", size=25)
 plt.xlabel('False Positive Rate',size=20); plt.xticks(size=15)
 plt.ylabel('True Positive Rate',size=20); plt.yticks(size=15)
 ax1.legend(loc="lower right",prop={"size":15})
 ax2 = fig.add_subplot(122)
 for i in range(FOLDS):
ax2.step(recall[i], precision[i], where='post', color=colors[i],
label=r'AP Fold %i(AP = %0.2f)' %(i+1,average_precision[i]),
lw=3, alpha=.5)
 ax2.step(recall['mean'], precision['mean'], where='post', color='b',
label=r'Mean AP(AP = %0.2f)' %(average_precision['mean']),
lw=4, alpha=.8)
 ax2.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])
 plt.title("Precision Recall Curve", size=25)
 plt.xlabel('Recall',size=20); plt.xticks(size=15)
 plt.ylabel('Precision',size=20); plt.yticks(size=15)
 ax2.legend(loc="lower left",prop={"size":15})
 plt.show()
xg_boost.fit(X_train, Y_train)
 Y_pred = xg_boost.predict(X_test)
 print(xg_boost.score(X_train, Y_train))
scores = model_selection.cross_val_score(xg_boost, X_train, Y_train, cv=5, scoring='accuracy')
 print(scores)
 print("Kfold on XGBClassifier: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std()))<save_to_csv>
test_ds =(load_dataset(files_test, labeled=False, return_imgname=True, ordered=True)
.batch(BATCH_SIZES[fold])
.prefetch(AUTO))
image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(test_ds.unbatch())])
 submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name' ).reset_index(drop=True)
 submission.to_csv('submission_noensemble.csv', index=False)
 submission.head()
submission = pd.DataFrame({
"PassengerId": data[train_size:]["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission.csv', index=False )<set_options>
train = pd.read_csv('/kaggle/input/melanomaextendedtabular/external_upsampled_tabular.csv')
 train.head()
!pip install fastai==0.7.0
%load_ext autoreload
%autoreload 2
%matplotlib inline<load_from_csv>
train['anatom_site_general_challenge'].replace(['anterior torso','lateral torso','posterior torso'],
'torso', inplace=True )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df.head()<concatenate>
train.isnull().sum()
train_cats(train_df)
 apply_cats(test_df, train_df )<split>
train.isnull().sum()
df_trn, y_trn, nas = proc_df(train_df, 'Survived')
 df_test, _, _ = proc_df(test_df, na_dict=nas)
 df_trn.head()<drop_column>
def fillna(df, column):
na_idx = df[df[column].isnull() ].index.tolist()
prob = df[column].value_counts(normalize=True ).sort_index().tolist()
for i in na_idx:
df.iloc[i, df.columns.get_loc(column)] = choices(sorted(df[column].dropna().unique().tolist()), prob)[0]
train.sex.replace('unknown', np.nan, inplace=True)
 fillna(train, 'sex')
 fillna(train, 'anatom_site_general_challenge')
 fillna(test, 'anatom_site_general_challenge')
 train['age_approx'] = train['age_approx'].fillna(round(train['age_approx'].mean()))
print('Train Missing Data Count: %i, Test Missing Data Count: %i'%(train.isnull().sum().sum() , test.isnull().sum().sum()))
df_trn.drop(['Age_na'], axis =1, inplace = True)
 df_test.drop(['Age_na', 'Fare_na'], axis =1, inplace = True)
 df_test.head()<compute_test_metric>
train = pd.get_dummies(train, columns=["anatom_site_general_challenge"], prefix='site',
drop_first=True)
 train.replace({'sex': {'female':0, 'male': 1}}, inplace=True)
 train.head()
def rmse(x,y): return math.sqrt(((x-y)**2 ).mean())
 def print_score(m):
res = [rmse(m.predict(train_X), train_y), rmse(m.predict(val_X), val_y),
m.score(train_X, train_y), m.score(val_X, val_y)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<split>
test = pd.get_dummies(test, columns=["anatom_site_general_challenge"], prefix='site',
drop_first=True)
 test.replace({'sex': {'female':0, 'male': 1}}, inplace=True)
 test.head()
train_X, val_X, train_y, val_y = train_test_split(df_trn, y_trn, test_size=0.33, random_state=42 )<train_model>
y_train = train['target']
X_train = train.drop(['image_name','height','width','target'],
axis=1)
 X_test = test.drop(['image_name','patient_id'],axis=1)
 scaler = StandardScaler()
X_train_s = X_train.copy()
X_train_s[['age_approx']] = scaler.fit_transform(X_train_s[['age_approx']])
 X_test_s = X_test.copy()
X_test_s[['age_approx']] = scaler.fit_transform(X_test_s[['age_approx']])
 X_train_s.head()
%time
m = RandomForestClassifier(n_estimators=1, min_samples_leaf=10, n_jobs=-1, max_depth = 3, oob_score=True)
 m.fit(train_X, train_y)
 print_score(m )<train_model>
lr = LogisticRegression(penalty='l2', class_weight='balanced', random_state=SEED)
 nb = GaussianNB()
rf = RandomForestClassifier(n_estimators=1000, max_depth=2, class_weight='balanced',
n_jobs=-1, random_state=SEED)
 estimators = [lr, rf, nb]
cv = StratifiedKFold(5, shuffle=True, random_state=SEED)
 def model_cv(X_train, y_train, estimators, cv):
for est in estimators:
model_name = est.__class__.__name__
cv_results = cross_validate(est, X_train, y_train, cv=cv,
scoring='roc_auc',
return_train_score=True,
n_jobs=-1)
 train_score = cv_results['train_score'].mean()
test_score = cv_results['test_score'].mean()
print('%s - Train Score: %.3f, Val Score: %.3f'%(model_name,train_score,test_score))
model_cv(X_train_s, y_train, estimators, cv )
%time
m = RandomForestClassifier(n_estimators=20, min_samples_leaf=10, max_features=0.7, n_jobs=-1, oob_score=True)
 m.fit(train_X, train_y)
 print_score(m )<train_model>
def model_blend(X_train, y_train, X_test, estimators):
mean_prob = 0
for est in estimators:
est.fit(X_train, y_train)
 mean_prob += est.predict_proba(X_test)[:,1]
return mean_prob/len(estimators)
 meta_df = pd.DataFrame(columns=['image_name', 'target'])
 meta_df['image_name'] = sample['image_name']
meta_df['target'] = model_blend(X_train_s, y_train, X_test_s, estimators)
 meta_df.to_csv('meta_pred.csv', header=True, index=False)
 meta_df.head()
<split><EOS> 
submission_effnet_ensemble = pd.read_csv('.. /input/effnet-ensemble/submission_effnet_ensemble.csv')
 submission.target =(submission_effnet_ensemble.target)+(meta_df.target * 0.1)
 submission.to_csv('submission.csv', index=False)
 submission.head()