Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<set_options>
pd.options.display.max_columns = None
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 potential_features = ['ps_ind_06_bin',
'ps_ind_07_bin',
'ps_ind_08_bin',
'ps_ind_16_bin',
'ps_ind_17_bin',
'ps_car_08_cat',
'ps_ind_04_cat',
'ps_car_03_cat',
'ps_car_11_cat',
'ps_car_09_cat',
'ps_car_06_cat',
'ps_ind_05_cat',
'ps_car_05_cat',
'ps_car_04_cat',
'ps_car_01_cat',
'ps_car_02_cat',
'ps_ind_02_cat',
'ps_car_07_cat',
'ps_car_13',
'ps_car_12',
'ps_reg_02',
'ps_reg_03',
'ps_car_15',
'ps_reg_01',
'ps_ind_15',
'ps_ind_01',
'ps_car_14',
'ps_ind_03',
'ps_ind_14']
train_df = train_df[['target'] + potential_features]
test_ids = test_df['id']
test_df = test_df[potential_features]
!rm -rf darknet images labels metadata backup cfg
!rm -rf train_log.txt darknet53.conv.74 darknet.py darknet_gpu
!rm -rf test.jpg
!rm -rf __pycache__.ipynb_checkpoints<import_modules>
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
over_factor = 0.6
row_count = round(len(train_df[train_df['target'] == 1])* over_factor)
 new_rows = train_df[train_df['target'] == 1].sample(row_count, replace=True, random_state=1)
 train_df = train_df.append(new_rows, ignore_index=True)
 train_df['target'].value_counts(1 )
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
train_miss = []
test_miss = []
print("train columns with missing data: ")
 for col in train_df.columns:
if(train_df[col].min() == -1)and('cat' not in col):
train_miss.append(col)
 print(col)
 print("
test columns with missing data: ")
 for col in test_df.columns:
if(test_df[col].min() == -1)and('cat' not in col):
test_miss.append(col)
 print(col )
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
for col in train_miss:
print("Train: ", col, ": ", len(train_df[train_df[col] == -1]))
for col in test_miss:
print("Test: ", col, ": ", len(test_df[test_df[col] == -1]))
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=512, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.6:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.55
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
train_df.loc[train_df['ps_car_12'] == -1, 'ps_car_12'] = train_df[train_df['ps_car_12'] != -1]['ps_car_12'].mean()
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(( 2, 2))(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.95 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.95 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=3):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
feature_mod = linear_model.LinearRegression()
feature_xgb = XGBRegressor(n_estimators=200)
 corr_list = ['ps_car_12', 'ps_car_13']
feature_mod.fit(train_df[train_df['ps_car_14'] != -1][corr_list], train_df[train_df['ps_car_14'] != -1]['ps_car_14'])
 train_df.loc[train_df['ps_car_14'] == -1, 'ps_car_14'] = feature_mod.predict(train_df[train_df['ps_car_14'] == -1][corr_list])
 test_df.loc[test_df['ps_car_14'] == -1, 'ps_car_14'] = feature_mod.predict(test_df[test_df['ps_car_14'] == -1][corr_list])
 corr_list = ['ps_reg_02', 'ps_car_13', 'ps_car_12', 'ps_ind_01']
feature_xgb.fit(train_df[train_df['ps_reg_03'] != -1][corr_list], train_df[train_df['ps_reg_03'] != -1]['ps_reg_03'])
 train_df.loc[train_df['ps_reg_03'] == -1, 'ps_reg_03'] = feature_xgb.predict(train_df[train_df['ps_reg_03'] == -1][corr_list])
 test_df.loc[test_df['ps_reg_03'] == -1, 'ps_reg_03'] = feature_xgb.predict(test_df[test_df['ps_reg_03'] == -1][corr_list] )
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 25
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=25, shuffle=True )<predict_on_test>
cat_cols = [x for x in potential_features if '_cat' in x ]
cat_cols = list(set(cat_cols)- set(['ps_car_11_cat']))
train_df = pd.get_dummies(data=train_df, columns=cat_cols, drop_first=True)
 test_df = pd.get_dummies(data=test_df, columns=cat_cols, drop_first=True )
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<import_modules>
columns_needed = max(train_df['ps_car_11_cat'].max().item().bit_length() ,
test_df['ps_car_11_cat'].max().item().bit_length())
 format_string = '0>'+ str(columns_needed)+ 'b'
bin_cols = train_df['ps_car_11_cat'].apply(lambda x: format(x, format_string)).str.extractall('(\d)' ).unstack().astype(np.int8 ).add_prefix('ps_car_11_cat_b')
 bin_cols.columns = bin_cols.columns.droplevel()
train_df = pd.concat([train_df, bin_cols], axis=1)
 train_df.drop('ps_car_11_cat', inplace=True, axis=1)
 bin_cols = test_df['ps_car_11_cat'].apply(lambda x: format(x, format_string)).str.extractall('(\d)' ).unstack().astype(np.int8 ).add_prefix('ps_car_11_cat_b')
 bin_cols.columns = bin_cols.columns.droplevel()
test_df = pd.concat([test_df, bin_cols], axis=1)
 test_df.drop('ps_car_11_cat', inplace=True, axis=1 )
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
MAX_ROUNDS = 400
LEARNING_RATE = 0.07
model = XGBClassifier(
 n_estimators=MAX_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
gamma=10,
reg_alpha=8,
reg_lambda=1.3
)
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 y_valid_preds = 0 * train_df['target']
y_test_preds = 0
for i,(train_index, valid_index)in enumerate(kf.split(train_df)) :
print("
Starting fold {}".format(i+1))
start = time.time()
y_train = train_df['target'].loc[train_index]
X_train = train_df.drop('target', axis=1 ).loc[train_index]
y_valid = train_df['target'].loc[valid_index]
X_valid = train_df.drop('target', axis=1 ).loc[valid_index]
fit_model = model.fit(X_train, y_train, verbose=True)
 preds = fit_model.predict_proba(X_valid)[:, 1]
gini = eval_gini(y_valid, preds)
 y_valid_preds.loc[valid_index] = preds
y_test_preds += fit_model.predict_proba(test_df)[:, 1]
print("Fold {} Gini score: {}".format(i, gini))
print("Completed fold {} in {:.2f} minutes
".format(i+1,(time.time() - start)/60))
y_test_preds /= K
print("
Gini for full training set:")
 eval_gini(train_df['target'], y_valid_preds )
<define_variables><EOS> 
sub = pd.DataFrame()
sub['id'] = test_ids
sub['target'] = y_test_preds
sub.to_csv('xgb_submit_3.csv', float_format='%.6f', index=False )