Kaggle competition: Titanic - Machine Learning from Disaster
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
dataframe = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
sns.countplot(dataframe['Survived'], hue = dataframe['Sex'])
 Dead, lives = dataframe.Survived.value_counts()
male, female = dataframe.Sex.value_counts()
print("Percentage of Male on ship:", round(male/(male+female)*100))
print("Percentage of Female on ship:", round(female/(male+female)*100))
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
dataframe.Pclass.unique()
dataframe.Pclass.value_counts()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
sns.countplot(dataframe['Embarked'], hue = dataframe['Survived'] )
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
dataframe.isnull().values.any()
dataframe.isnull().sum()
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
test.isnull().sum()
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.000020 * REPLICAS * batch_size/16
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 return lr_callback<train_model>
dataframe.isnull().sum()
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=42)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Loading model...')
 model.load_weights('.. /input/melanoma-weights/fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=1)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=1)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 print('
print()<save_to_csv>
test.isnull().sum()
oof = np.concatenate(oof_pred)
 names = np.concatenate(oof_names)
 true = np.concatenate(oof_tar)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(image_name = names, pred = oof, target=true))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
test.isnull().sum()
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
train_set = dataframe.drop(['Name','Cabin', 'Ticket','PassengerId', ], axis = 1)
 test_set = test.drop(['Name','Cabin', 'Ticket', 'PassengerId', ], axis = 1)
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
test_set.isnull().sum()
!pip install -q efficientnet
!pip install -q git+https://github.com/AmedeoBiolatti/dsqol<import_modules>
train_set['Embarked'].fillna(train_set['Embarked'].mode() [0], inplace = True)
import os, re, time, tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics, model_selection
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow import keras
from tensorflow.keras import backend as K
from efficientnet import tfkeras as efnet
from kaggle_datasets import KaggleDatasets<import_modules>
y = train_set.iloc[:, 0].values
X = train_set.iloc[:, train_set.columns != 'Survived'].values
print(X[0] )
from dsqol.tf import imgaug
from dsqol.tf.data import balance
from dsqol.tf.utils import average
from dsqol.tf import losses<define_variables>
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [6])], remainder='passthrough')
 X = np.array(ct.fit_transform(X))
print(X[2] )
SEED = 42
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 TIME_BUDGET = 2.5 * 3600
FOLDS = 5
INCLUDE_2019 = 0
INCLUDE_2018 = 1
INCLUDE_MALIGNANT = 1
IMG_READ_SIZE = 384
IMG_SIZE = 384
BALANCE_POS_RATIO = False
EFF_NET = 5
LOSS_TYPE = 'BCE'
LOSS_PARAMS = dict(label_smoothing=0.05)
 BATCH_SIZE = 32
EPOCHS = 10
TBM = 6
TTA = 20
VALID_FREQ = 1
N_SWA = 3
SWA_DECAY = 0.9<train_on_grid>
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [6])], remainder='passthrough')
 test_set = np.array(ct.fit_transform(test_set))
print(test_set[1])
DEVICE = "TPU"
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
GLOBAL_BATCH_SIZE = BATCH_SIZE * REPLICAS
print("REPLICAS: %d" % REPLICAS )<define_variables>
le = LabelEncoder()
X[:, 4 ] = le.fit_transform(X[:,4])
 print(X[1])
GCS_PATH1 = KaggleDatasets().get_gcs_path('melanoma-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))<load_from_csv>
le = LabelEncoder()
test_set[:, 4] = le.fit_transform(test_set[:,4])
 print(test_set[2] )
df_base_train = pd.read_csv(".. /input/siim-isic-melanoma-classification/train.csv")
 df_base_test = pd.read_csv(".. /input/siim-isic-melanoma-classification/test.csv" )<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20 )
train_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "train*.tfrec"))
if INCLUDE_2019:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(1, 30, 2)])
 if INCLUDE_2018:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(0, 30, 2)])
 if INCLUDE_MALIGNANT:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH3, "train%.2i*.tfrec" % i)for i in range(15, 30, 1)])
 print("%d train files found" % len(train_files))<train_model>
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test =sc.transform(X_test )
test_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "test*.tfrec"))
print("%d test files found" % len(test_files))<prepare_x_and_y>
rfc=RandomForestClassifier()
parameters= {'n_estimators':[ 100,200,300,400, 600],
'max_depth':[3,4,6,7],
'criterion':['entropy','gini']
}
rfc=GridSearchCV(rfc, param_grid=parameters, cv = 5)
 rfc.fit(X_train,y_train)
 print("The best value of leanring rate is: ",rfc.best_params_, )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name=True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
rf_model = RandomForestClassifier(criterion= 'gini', n_estimators = 100 ,max_depth = 6, random_state = 0)
 rf_model.fit(X_train, y_train)
 y_pred = rf_model.predict(X_test)
 Random_forest_acc= accuracy_score(y_test, y_pred)
 print('acc = ', Random_forest_acc)
def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0):
return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
def ClasswiseMixup(probability: float=1.0, batch_size: int = 16, alpha: float=1.0, beta: float=1.0):
def classwise_mixup(img, tar):
do = tf.cast(tf.random.uniform([batch_size], 0, 1)< probability, tf.float32)
 gamma = tf.reshape(tf.random.uniform([batch_size], 0, 1)* do, [-1, 1, 1, 1])
 shuffled_img = tf.random.shuffle(img)
 img_new =(1 - gamma)* img + gamma * shuffled_img
return img_new
return classwise_mixup<statistical_test>
model = LogisticRegression()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 LogisticReg_acc= accuracy_score(y_test, y_pred)
 print('acc = ', LogisticReg_acc )
AUG_BS = 64
def base_aug(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img
dropout_aug = lambda img, o: dropout(img, DIM=IMG_READ_SIZE, PROBABILITY=0.75, CT=8, SZ=0.15)
 transform_aug = imgaug.Transform(dim=IMG_READ_SIZE, hzoom_mult=10.0, wzoom_mult=10.0)
 cw_mixup_aug = ClasswiseMixup(probability=0.1, batch_size=AUG_BS)
 def basic_augmentation_pipeline(ds: tf.data.Dataset, dim=None, batch_size=None)-> tf.data.Dataset:
ds = ds.map(lambda i, o:(transform_aug(i), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(base_aug(i), o), num_parallel_calls=AUTO)
 return ds<create_dataframe>
model = SVC(kernel = 'rbf', random_state = 0)
 model.fit(X, y)
 y_pred = model.predict(X_test)
 SVC_acc = accuracy_score(y_test, y_pred)
 print('acc = ', SVC_acc )
def get_dataset(files, augment=False, repeat=False, shuffle=False, labeled=True, batch_size=16, drop_remainder=False,
dim=256, read_dim=None 
)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024 * 8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds
def get_balanced_dataset(files, augment=False, cw_augment=False, repeat=False, shuffle=False, batch_size=16, drop_remainder=False,
dim=256, read_dim=None, pos_ratio=False)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds0, ds1 = balance.separate_by_target(ds)
 ds0 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 ds1 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 if cw_augment:
ds0 = ds0.batch(AUG_BS, drop_remainder=drop_remainder); ds1 = ds1.batch(AUG_BS, drop_remainder=drop_remainder)
 for a in cw_augment:
ds0 = ds0.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds1 = ds1.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds0 = ds0.unbatch() ; ds1 = ds1.unbatch()
ds = balance.merge_ds(ds0, ds1, pos_ratio)
 if shuffle:
ds = ds.shuffle(1024)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(lambda i, o:(a(i, o), o), num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
model = GaussianNB()
model.fit(X, y)
 y_pred = model.predict(X_test)
 Gaussian_acc = accuracy_score(y_test, y_pred)
 print('acc = ', Gaussian_acc )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 pass<create_dataframe>
model = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)
 model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 DT_acc = accuracy_score(y_test, y_pred)
 print('acc = ', DT_acc)
show_dataset(128, 8, 2, get_balanced_dataset(train_files, augment=[dropout_aug], cw_augment=[cw_mixup_aug] ).take(10 ).unbatch() )<choose_model_class>
gbc=GradientBoostingClassifier()
parameters= {'n_estimators':[ 50,100,200,300, ],
'max_depth':[3,4,6,7]
}
gbreg=GridSearchCV(gbc, param_grid=parameters, cv = 5)
 gbreg.fit(X_train,y_train)
 print("The best value of leanring rate is: ",gbreg.best_params_, )
def build_model(dim=128, ef=0):
inp = keras.layers.Input(shape=(dim,dim,3))
base = getattr(efnet, 'EfficientNetB%d' % ef )(input_shape=(dim, dim, 3), weights='imagenet', include_top=False)
 x = base(inp)
 x = keras.layers.GlobalAveragePooling2D()(x)
 x = keras.layers.Dense(1 )(x)
 x = keras.layers.Activation('sigmoid', dtype='float32' )(x)
 model = keras.Model(inputs=inp,outputs=x)
 opt = keras.optimizers.Adam(learning_rate=1e-3)
 if LOSS_TYPE.upper() == 'BCE':
loss = keras.losses.BinaryCrossentropy(**LOSS_PARAMS)
 elif LOSS_TYPE.upper() == 'FOCAL':
loss = losses.BinaryFocalLoss(**LOSS_PARAMS)
 model.compile(optimizer=opt, loss=loss, metrics=['AUC'])
 return model<init_hyperparams>
model_gb = GradientBoostingClassifier(n_estimators = 100, max_depth =4, random_state = 42)
 model_gb.fit(X_train, y_train)
 y_pred = model_gb.predict(X_test)
 GB_acc = accuracy_score(y_test, y_pred)
 print('acc = ', GB_acc )
mult = 1
lr_start = 5e-6
lr_max = 1.25e-6 * GLOBAL_BATCH_SIZE
lr_min = 1e-6
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr * mult
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('LR')
 plt.subplot(1, 2, 2)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('Log LR')
 plt.yscale('log');<load_from_csv>
classifier = XGBClassifier()
classifier.fit(X_train, y_train)
 y_pred = classifier.predict(X_test)
 xgb_acc = accuracy_score(y_pred, y_test)
 print('acc=',xgb_acc )
CKPT_FOLDER = ".. /working/ckpt"
if not os.path.exists(CKPT_FOLDER):
os.mkdir(CKPT_FOLDER)
 folds = list(model_selection.KFold(n_splits=FOLDS, shuffle=True, random_state=SEED ).split(np.arange(15)))
 testiness = pd.read_csv(".. /input/spicv-spicy-vi-make-your-cv-more-testy/testiness.csv")
 TOTAL_POS = 581 + 2858 * INCLUDE_2019 + 1651 * INCLUDE_2018 + 580 * INCLUDE_MALIGNANT<create_dataframe>
print('RF_acc=', Random_forest_acc)
 print('Logistic_acc=', LogisticReg_acc)
 print('SVC_acc=', SVC_acc)
 print('Gaussian_acc=', Gaussian_acc)
 print('DecisionTree_acc=', DT_acc)
 print('GradBoost_acc=', GB_acc)
 print('XGBoost_acc=', xgb_acc)
VERBOSE = 1
PLOT = 1
histories = []
df_oof = pd.DataFrame() ; df_res = pd.DataFrame()
t_start = time.time()
for fold,(idTrain, idValid)in enumerate(folds):
print("
print(( "
print("
if DEVICE == 'TPU':
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 fold_valid_files = [f for f in train_files if any([int(re.match("^train([0-9]+)", f.split("/")[-1] ).group(1)) % 15 == i for i in idValid])]
fold_valid_files = [f for f in fold_valid_files if GCS_PATH1 in f]
fold_train_files = [f for f in train_files if f not in fold_valid_files]
np.random.shuffle(fold_train_files)
 print("Train files: %d\t\t Valid files: %d" %(len(fold_train_files), len(fold_valid_files)))
 K.clear_session()
if DEVICE == 'TPU':
keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
 with strategy.scope() :
model = build_model(dim=IMG_SIZE, ef=EFF_NET)
 FOLD_CKPT_FOLDER = os.path.join(CKPT_FOLDER, "fold%d" % fold)
 if not os.path.exists(FOLD_CKPT_FOLDER):
os.mkdir(FOLD_CKPT_FOLDER)
 callbacks =[
keras.callbacks.ModelCheckpoint(os.path.join(FOLD_CKPT_FOLDER, "model_fold%d_e{epoch:02d}.h5" % fold), save_weights_only=True),
keras.callbacks.LearningRateScheduler(lrfn)
 ]
if BALANCE_POS_RATIO:
print("Using balanced dataset with pos_ratio = %d%%" % int(100 * BALANCE_POS_RATIO))
ds_train = get_balanced_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
pos_ratio=BALANCE_POS_RATIO,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 FOLD_POS = TOTAL_POS *(FOLDS - 1)/ FOLDS
STEPS = int(FOLD_POS / BALANCE_POS_RATIO / GLOBAL_BATCH_SIZE)
 else:
print("Using unbalanced dataset")
 ds_train = get_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 STEPS = int(count_data_items(fold_train_files)/ GLOBAL_BATCH_SIZE)
 ds_valid = get_dataset(fold_valid_files, repeat=False, augment=False, drop_remainder=False, shuffle=False,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM)
 print("Training...")
 history = model.fit(
 ds_train,
validation_data = ds_valid,
epochs = EPOCHS,
steps_per_epoch = STEPS,
verbose = VERBOSE,
callbacks = callbacks,
validation_freq = VALID_FREQ 
)
 histories.append(history)
 ckpt_files = np.sort(tf.io.gfile.glob(os.path.join(FOLD_CKPT_FOLDER, "*.h5")))
 ckpt_files_fow_swa = ckpt_files[-N_SWA:]
if len(ckpt_files_fow_swa)> 1:
with strategy.scope() :
model = average.average_weights(ckpt_files_fow_swa, decay=SWA_DECAY, model=model)
 for f in ckpt_files:
os.remove(f)
 model.save(os.path.join(CKPT_FOLDER, "model_fold%d.h5" % fold))
ds_valid = get_dataset(fold_valid_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True)
 ct_valid = count_data_items(fold_valid_files); STEPS = int(np.ceil(TTA * ct_valid / GLOBAL_BATCH_SIZE / TBM))
fold_valid_pred = model.predict(ds_valid, steps=STEPS, verbose=1)
 fold_valid_pred = fold_valid_pred[:ct_valid * TTA,]
ds_valid = get_dataset(fold_valid_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_valid_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_valid.map(lambda i, n: n)], 0)
 fold_df = pd.DataFrame({'image_name': np.tile(fold_valid_names, [TTA]), 'pred': fold_valid_pred.squeeze() , 'fold': fold})
 df_oof = pd.concat([df_oof, fold_df])
 fold_df['image_name'] = fold_df['image_name'].str.replace('_downsampled', '')
 fold_df = fold_df.groupby('image_name' ).mean().reset_index()
fold_df = fold_df.merge(df_base_train[['image_name', 'patient_id', 'target']], on='image_name' ).merge(testiness, on='image_name')
 fold_df['fold'] = fold
auc = metrics.roc_auc_score(fold_df.target, fold_df.pred)
 ds_test = get_dataset(test_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True, labeled=False)
 ct_test = count_data_items(test_files); STEPS = int(np.ceil(TTA * ct_test / GLOBAL_BATCH_SIZE / TBM))
fold_test_pred = model.predict(ds_test.map(lambda i, l: i), steps=STEPS, verbose=1)
 fold_test_pred = fold_test_pred[:ct_test * TTA,]
ds_test = get_dataset(test_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_test_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_test.map(lambda i, n: n)], 0)
 fold_res = pd.DataFrame({'image_name': np.tile(fold_test_names, [TTA]), 'pred': fold_test_pred.squeeze() , 'fold': fold})
 df_res = pd.concat([df_res, fold_res])
 used_time_till_now = time.time() - t_start
time_per_fold = used_time_till_now /(fold + 1)
 print("Validation AUC last epoch = %.4f" % history.history['val_auc'][-1])
 print("Validation AUC(TTA %2d)= %.4f" %(TTA, auc))
print("Total time = %ds\t\tTime per fold = %ds" %(int(used_time_till_now), int(time_per_fold)))
 if PLOT:
plt.figure(figsize=(16, 4))
plt.subplot(1, 2, 1)
 plt.plot(history.history['loss'], color='tab:blue', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_loss'], color='tab:blue', marker='x', linestyle=':')
 plt.yscale('log')
 plt.subplot(1, 2, 2)
 plt.plot(history.history['auc'], color='tab:red', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_auc'], color='tab:red', marker='x', linestyle=':')
 plt.show()
del model, ds_train, ds_valid, ds_test
print("
")
 if(fold + 1)< FOLDS:
time_till_next_fold = used_time_till_now + time_per_fold
if time_till_next_fold > TIME_BUDGET:
print("Forecasted total time till next fold completion = %d(budget = %d)" %(time_till_next_fold, TIME_BUDGET))
break
pass<merge>
rf_model = RandomForestClassifier(criterion= 'gini', n_estimators = 100 ,max_depth = 6, random_state = 0)
 rf_model.fit(X, y)
 final_pred = rf_model.predict(test_set)
 final_pred
xxx = df_oof.groupby('image_name' ).mean().reset_index().merge(df_base_train, on='image_name')
 print("OOF AUC(TTA %d)= %.4f" %(TTA, metrics.roc_auc_score(xxx.target, xxx.pred)) )<save_to_csv>
survivors = pd.DataFrame(final_pred, columns = ['Survived'])
 len(survivors)
 survivors.insert(0, 'PassengerId', test['PassengerId'], True)
 survivors
df_res.to_csv('.. /working/test_res_all.csv', index=False)
 df_oof.to_csv('.. /working/oof_res_all.csv', index=False )<save_to_csv>
survivors.to_csv('Submission.csv', index = False )
df_res[['image_name', 'pred']].groupby('image_name' ).mean().reset_index().rename({'pred': 'target'}, axis=1 ).to_csv("submission.csv", index=False )<load_from_csv>
dataframe = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(train["Address"])+ list(test["Address"]))
haha = tokenizer.texts_to_sequences(train["Address"])
 haha2 = tokenizer.texts_to_sequences(test["Address"])
 padded = pad_sequences(haha, maxlen=7)
 padded2 = pad_sequences(haha2, maxlen=7)
 model = Sequential()
model.add(Embedding(2201, 1, input_length=7))
model.add(Flatten())
 model.add(Dense(39, activation='softmax'))
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc'])
 le2 = LabelEncoder()
y= le2.fit_transform(train['Category'])
 model.fit(padded, y, epochs = 10 , batch_size = 2048)
 preds = model.predict(padded)
 preds2 = model.predict(padded2)
 pca1 = PCA(n_components=2)
 X_low = pca1.fit_transform(preds)
 X_low2 = pca1.transform(preds2)
 train['Date'] = train['Dates'].dt.date
train['n_days'] =(train['Date'] - train['Date'].min() ).apply(lambda x: x.days)
 train['Day'] = train['Dates'].dt.day
train['DayOfWeek'] = train['Dates'].dt.weekday
train['Month'] = train['Dates'].dt.month
train['Year'] = train['Dates'].dt.year
train['Hour'] = train['Dates'].dt.hour
train['Minute'] = train['Dates'].dt.minute
train['Block'] = train['Address'].str.contains('block', case=False)
 train['ST'] = train['Address'].str.contains('ST', case=False)
 train["X_Y"] = train["X"] - train["Y"]
train["XY"] = train["X"] + train["Y"]
test['Date'] = test['Dates'].dt.date
test['n_days'] =(test['Date'] - test['Date'].min() ).apply(lambda x: x.days)
 test['Day'] = test['Dates'].dt.day
test['DayOfWeek'] = test['Dates'].dt.weekday
test['Month'] = test['Dates'].dt.month
test['Year'] = test['Dates'].dt.year
test['Hour'] = test['Dates'].dt.hour
test['Minute'] = test['Dates'].dt.minute
test['Block'] = test['Address'].str.contains('block', case=False)
 test['ST'] = test['Address'].str.contains('ST', case=False)
 test["X_Y"] = test["X"] - test["Y"]
test["XY"] = test["X"] + test["Y"]
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
y= le2.fit_transform(train['Category'])
 train = pd.concat([train, pd.DataFrame(X_low)], 1)
 test = pd.concat([test, pd.DataFrame(X_low2)], 1)
 le3 = LabelEncoder()
le3.fit(list(train['Address'])+ list(test['Address']))
train['Address'] = le3.transform(train['Address'])
 test['Address'] = le3.transform(test['Address'])
 train.drop(['Dates','Date','Descript','Resolution', 'Category'], 1, inplace=True)
 test.drop(['Dates','Date',], 1, inplace=True)
 train.to_csv("train.csv", index=False)
 test.to_csv("test.csv", index=False)
 hyper = {'colsample_bytree': 0.625,
'is_unbalance': False,
'learning_rate': 0.025,
'min_child_samples': 105,
'num_class': 39,
'num_leaves': 233,
'objective': 'multiclass',
'reg_alpha': 0.4000134592012641,
'reg_lambda': 0.5082596745249518,
'subsample': 0.9338693244190213,
'subsample_for_bin': 140000,
'n_estimators': 600}
model = LGBMClassifier(**hyper)
 model.fit(train, y, categorical_feature=["PdDistrict", "DayOfWeek"])
 preds = model.predict_proba(test)
 submission = pd.DataFrame(preds, columns=le2.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission.to_csv('LGBM_final.csv', index_label='Id' )<import_modules>
sns.countplot(dataframe['Survived'], hue = dataframe['Sex'])
 Dead, lives = dataframe.Survived.value_counts()
male, female = dataframe.Sex.value_counts()
print("Percentage of Male on ship:", round(male/(male+female)*100))
print("Percentage of Female on ship:", round(female/(male+female)*100))
For the Traveling Santa 2018 competition.All computation is done
within the Kernel, including building the list of primes and creating the TSPLIB file.
 The downloaded C codes(in gzipped tar files)are left in the Kernel.
 Note: The prime_thread code, built on top of the linkern code from Concorde, uses as a
random seed the current time.The behavior of the Kernel is random .<load_from_csv>
dataframe.Pclass.unique()
dataframe.Pclass.value_counts()
cities = pd.read_csv('.. /input/cities.csv', index_col=['CityId'], nrows=None)
 cities_1000 = cities * 1000<save_to_csv>
sns.countplot(dataframe['Embarked'], hue = dataframe['Survived'] )
def write_tsp(nodes, filename, name='Santa Prime Paths'):
with open(filename, 'w')as f:
f.write('NAME : %s
' % name)
 f.write('COMMENT : %s
' % name)
 f.write('TYPE : TSP
')
 f.write('DIMENSION : %d
' % len(nodes))
f.write('EDGE_WEIGHT_TYPE : EUC_2D
')
 f.write('NODE_COORD_SECTION
')
 for row in nodes.itertuples() :
f.write('%d %.11f %.11f
' %(row.Index + 1, row.X, row.Y))
f.write('EOF
')
 write_tsp(cities_1000, 'santa197769.tsp' )<set_options>
dataframe.isnull().values.any()
dataframe.isnull().sum()
%%bash
rm -r LKH-2.0.9
rm LKH-2.0.9.t*
wget http://akira.ruc.dk/~keld/research/LKH/LKH-2.0.9.tgz<set_options>
test.isnull().sum()
%%bash
tar xzvf LKH-2.0.9.tgz
cd LKH-2.0.9
make
mv LKH..
cd..
rm -r LKH-2.0.9<set_options>
dataframe.isnull().sum()
%%bash 
./LKH par0.par<set_options>
test.isnull().sum()
%%bash 
./LKH par7.par<save_to_csv>
test.isnull().sum()
def write_xy(nodes, filename):
with open(filename, 'w')as f:
f.write('%d
' % len(nodes))
for row in nodes.itertuples() :
f.write('%.12f %.12f
' %(row.X, row.Y))
f.write('EOF
')
 write_xy(cities, 'kaggle.xy' )<set_options>
train_set = dataframe.drop(['Name','Cabin', 'Ticket','PassengerId', ], axis = 1)
 test_set = test.drop(['Name','Cabin', 'Ticket', 'PassengerId', ], axis = 1)
%%bash
wget http://www.math.uwaterloo.ca/tsp/pm/gen_primes.c
gcc -o gen_primes gen_primes.c -lm 
./gen_primes
head primes.txt
rm gen_primes*<set_options>
test_set.isnull().sum()
%%bash
wget http://www.math.uwaterloo.ca/tsp/pm/PM_1.tgz
tar xzvf PM_1.tgz
cd PM_1
make prime_thread
mv prime_thread..
cd..
rm -r PM_1<install_modules>
train_set['Embarked'].fillna(train_set['Embarked'].mode() [0], inplace = True)
%%bash
rm -r PM-LKH*
wget http://www.math.uwaterloo.ca/tsp/pm/PM-LKH-3b.tgz
tar xzvf PM-LKH-3b.tgz
mv PM-LKH-3b PM-LKH
cd PM-LKH
make
rm -r PMSRC_DIV<set_options>
y = train_set.iloc[:, 0].values
X = train_set.iloc[:, train_set.columns != 'Survived'].values
print(X[0] )
%%bash
cp kaggle.xy PM-LKH/
cp santa197769.tsp PM-LKH/
cp primes.txt PM-LKH/primes_list<load_from_csv>
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [6])], remainder='passthrough')
 X = np.array(ct.fit_transform(X))
print(X[2] )
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS<set_options>
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [6])], remainder='passthrough')
 test_set = np.array(ct.fit_transform(test_set))
print(test_set[1])
%%bash
cd PM-LKH 
./run_Segment_Optimization post.tour 5000 post2.tour<load_from_csv>
le = LabelEncoder()
X[:, 4 ] = le.fit_transform(X[:,4])
 print(X[1])
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS<set_options>
le = LabelEncoder()
test_set[:, 4] = le.fit_transform(test_set[:,4])
 print(test_set[2] )
%%bash
cd PM-LKH 
./run_Segment_Optimization post2.tour 7500 post3.tour<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20 )
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
rm -r DIV
rm -r DIV_TOURS<set_options>
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test =sc.transform(X_test )
%%bash
cd PM-LKH 
./run_Segment_Optimization post3.tour 6250 post4.tour<load_from_csv>
rfc=RandomForestClassifier()
parameters= {'n_estimators':[ 100,200,300,400, 600],
'max_depth':[3,4,6,7],
'criterion':['entropy','gini']
}
rfc=GridSearchCV(rfc, param_grid=parameters, cv = 5)
 rfc.fit(X_train,y_train)
 print("The best value of leanring rate is: ",rfc.best_params_, )
%%bash
cd PM-LKH
mv submission.csv.. /submission.csv
mv post4.tour.. /post4.tour
rm -r DIV
rm -r DIV_TOURS 
<train_model>
rf_model = RandomForestClassifier(criterion= 'gini', n_estimators = 100 ,max_depth = 6, random_state = 0)
 rf_model.fit(X_train, y_train)
 y_pred = rf_model.predict(X_test)
 Random_forest_acc= accuracy_score(y_test, y_pred)
 print('acc = ', Random_forest_acc)
Final run of the linkern-based code to search for global impprovemnts .<import_modules>
model = LogisticRegression()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 LogisticReg_acc= accuracy_score(y_test, y_pred)
 print('acc = ', LogisticReg_acc )
import numba
import numpy as np
import pandas as pd
from math import sqrt
from functools import lru_cache
from sklearn.neighbors import KDTree
from sympy import isprime, primerange
from tqdm import tqdm_notebook as tqdm
from ortools.constraint_solver import pywrapcp
from itertools import combinations, permutations
from scipy.spatial.distance import pdist, squareform
from ortools.constraint_solver import routing_enums_pb2
from sympy.utilities.iterables import multiset_permutations<load_from_csv>
model = SVC(kernel = 'rbf', random_state = 0)
 model.fit(X, y)
 y_pred = model.predict(X_test)
 SVC_acc = accuracy_score(y_test, y_pred)
 print('acc = ', SVC_acc )
cities = pd.read_csv('.. /input/reeindeer/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<choose_model_class>
model = GaussianNB()
model.fit(X, y)
 y_pred = model.predict(X_test)
 Gaussian_acc = accuracy_score(y_test, y_pred)
 print('acc = ', Gaussian_acc )
kdt = KDTree(XY )<load_from_csv>
model = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)
 model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 DT_acc = accuracy_score(y_test, y_pred)
 print('acc = ', DT_acc)
path = np.array(pd.read_csv('.. /input/reeindeer/path1515413.csv' ).Path)
<compute_test_metric>
gbc=GradientBoostingClassifier()
parameters= {'n_estimators':[ 50,100,200,300, ],
'max_depth':[3,4,6,7]
}
gbreg=GridSearchCV(gbc, param_grid=parameters, cv = 5)
 gbreg.fit(X_train,y_train)
 print("The best value of leanring rate is: ",gbreg.best_params_, )
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path)
 @numba.jit
def chunk_scores(chunk):
scores = np.zeros(10)
 pure_distance = 0
for i in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[i], chunk[i+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if is_not_prime[id_from]:
scores[9-i%10] += distance
scores *= 0.1
scores += pure_distance
return scores
@numba.jit('f8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:])', nopython=True, parallel=False)
 def score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes):
score = 0.0
last_city_id = head
for i in numba.prange(len(indexes)) :
index = indexes[i]
first, last, chunk_len = firsts[index], lasts[index], lens[index]
score += cities_distance(offset, last_city_id, first)
 score += scores[index,(offset + 1)% 10]
last_city_id = last
offset += chunk_len
return score + cities_distance(offset, last_city_id, tail)
 @numba.jit('i8(i8, i8, i8[:], i8[:], i8[:], i8, f8[:,:], i8[:,:], f8)', nopython=True, parallel=False)
 def best_score_permutation_index(offset, head, firsts, lasts, lens, tail, scores, indexes, best_score):
best_index = -1
for i in numba.prange(len(indexes)) :
score = score_compound_chunk(offset, head, firsts, lasts, lens, tail, scores, indexes[i])
 if score < best_score:
best_index, best_score = i, score
return best_index
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def sum_distance(ids):
res = 0
for i in numba.prange(len(ids)) :
for j in numba.prange(i + 1, len(ids)) :
res += cities_distance(0, ids[i], ids[j])
 return res
@lru_cache(maxsize=None)
 def indexes_permutations(n):
return np.array(list(map(list, permutations(range(n)))) )<define_search_space>
model_gb = GradientBoostingClassifier(n_estimators = 100, max_depth =4, random_state = 42)
 model_gb.fit(X_train, y_train)
 y_pred = model_gb.predict(X_test)
 GB_acc = accuracy_score(y_test, y_pred)
 print('acc = ', GB_acc )
trees = get_uplets(3, 5, 5 )<sort_values>
classifier = XGBClassifier()
classifier.fit(X_train, y_train)
 y_pred = classifier.predict(X_test)
 xgb_acc = accuracy_score(y_pred, y_test)
 print('acc=',xgb_acc )
path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for ids in tqdm(trees):
i1, i2, i3 = np.sort(path_index[ids])
 head, tail = path[i1-1], path[i3+1]
all_chunks = [[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1]]]
for i, chunks in enumerate(all_chunks):
chunks = [chunk for chunk in chunks if len(chunk)]
scores = np.array([chunk_scores(chunk)for chunk in chunks])
 lens = np.array([len(chunk)for chunk in chunks])
 firsts = np.array([chunk[0] for chunk in chunks])
 lasts = np.array([chunk[-1] for chunk in chunks])
 if i == 0:
best_score = score_compound_chunk(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) [0])
 index = best_score_permutation_index(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) , best_score)
 if index > 0:
perm = [chunks[i] for i in indexes_permutations(len(chunks)) [index]]
path[i1-1:i3+2] = np.concatenate([[head], np.concatenate(perm), [tail]])
 path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.3f}.Permutating path at indexes {i1}, {i2}, {i3}.')
 break
break<define_search_space>
print('RF_acc=', Random_forest_acc)
 print('Logistic_acc=', LogisticReg_acc)
 print('SVC_acc=', SVC_acc)
 print('Gaussian_acc=', Gaussian_acc)
 print('DecisionTree_acc=', DT_acc)
 print('GradBoost_acc=', GB_acc)
 print('XGBoost_acc=', xgb_acc)
fours = get_uplets(4, 5, 0 )<sort_values>
rf_model = RandomForestClassifier(criterion= 'gini', n_estimators = 100 ,max_depth = 6, random_state = 0)
 rf_model.fit(X, y)
 final_pred = rf_model.predict(test_set)
 final_pred
path_index = np.argsort(path[:-1])
 print(f'Total score is {score_path(path):.2f}.')
 for ids in tqdm(fours):
i1, i2, i3, i4 = np.sort(path_index[ids])
 head, tail = path[i1-1], path[i4+1]
all_chunks = [[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]],
[path[i1:i1+1], path[i1+1:i2][::-1], path[i2:i2+1], path[i2+1:i3][::-1], path[i3:i3+1], path[i3+1:i4][::-1], path[i4:i4+1]]
]
for i, chunks in enumerate(all_chunks):
chunks = [chunk for chunk in chunks if len(chunk)]
scores = np.array([chunk_scores(chunk)for chunk in chunks])
 lens = np.array([len(chunk)for chunk in chunks])
 firsts = np.array([chunk[0] for chunk in chunks])
 lasts = np.array([chunk[-1] for chunk in chunks])
 if i == 0:
best_score = score_compound_chunk(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) [0])
 index = best_score_permutation_index(i1-1, head, firsts, lasts, lens, tail, scores, indexes_permutations(len(chunks)) , best_score)
 if index > 0:
perm = [chunks[i] for i in indexes_permutations(len(chunks)) [index]]
path[i1-1:i4+2] = np.concatenate([[head], np.concatenate(perm), [tail]])
 path_index = np.argsort(path[:-1])
 print(f'New total score is {score_path(path):.3f}.Permutating path at indexes {i1}, {i2}, {i3}, {i4}.')
 break
break<create_dataframe>
survivors = pd.DataFrame(final_pred, columns = ['Survived'])
 len(survivors)
 survivors.insert(0, 'PassengerId', test['PassengerId'], True)
 survivors
<define_variables><EOS> 
survivors.to_csv('Submission.csv', index = False )