Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<init_hyperparams>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import random
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from mlxtend.classifier import StackingCVClassifier
LGB_PARAMS = {
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.1455,
'num_leaves': 129,
'min_child_samples': 78,
'max_depth': 13,
'subsample_freq': 1,
'subsample': 0.88,
'bagging_seed': 15,
'reg_alpha': 0.10107001,
'reg_lambda': 0.300132,
'colsample_bytree': 1.0
}<load_from_csv>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()
submission_csv = pd.read_csv(f'{DATA_PATH}/champs-scalar-coupling/sample_submission.csv', index_col='id' )<prepare_x_and_y>
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data<load_from_csv>
len(train_data[train_data['Pclass'] == 1]), len(train_data[train_data['Pclass'] == 2]), len(train_data[train_data['Pclass'] == 3] )
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=5, n_splits=5, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
model = LGBMRegressor(**LGB_PARAMS, n_estimators=8000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=1000, early_stopping_rounds=200)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
plt.scatter(y_val, y_val_pred)
 plt.show()
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score<find_best_params>
precentages = []
first = 136/216
seconds = 87/184
third = 119/491
precentages.append(first)
 precentages.append(seconds)
 precentages.append(third )
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 5
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score<create_dataframe>
percents = pd.DataFrame(precentages)
 percents.index += 1
pd.DataFrame({'type': list(cv_scores.keys()), 'cv_score': list(cv_scores.values())} )<save_to_csv>
train_data.isna().sum()
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )<save_to_csv>
test_data.isna().sum()
pd.read_csv('.. /input/sample_submission.csv',
converters = {'EncodedPixels': lambda p: None} ).to_csv('submission_paulorzp.csv', index=False )<save_to_csv>
df = [train_data,test_data]
for d in df:
d['Age'].fillna(d['Age'].median() ,inplace=True )
test_files = [f for f in os.listdir(".. /input/test/")]
df = pd.read_csv(".. /input/test_ship_segmentations.csv")
 df = df[df['ImageId'].isin(test_files)].drop_duplicates(subset="ImageId")
 df.to_csv("submission.csv", index=False)
 len(df )<import_modules>
train_data['Cabin'].value_counts()
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage<define_variables>
for d in df:
d['Cabin'].fillna('C',inplace=True )
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train/'
TEST = '.. /input/airbus-ship-detection/test/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection/ship_detection.csv'
exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']<choose_model_class>
train_data['Cabin'].isna().sum()
nw = 2
arch = resnet34<load_from_csv>
cabins = []
for i in train_data['Cabin']:
cabins.append(str(i))
train_names = [f for f in os.listdir(TRAIN)]
test_names = [f for f in os.listdir(TEST)]
for el in exclude_list:
if(el in train_names): train_names.remove(el)
 if(el in test_names): test_names.remove(el)
 tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId' )<categorify>
words = []
for i in cabins:
word = i[0]
words.append(word )
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )<categorify>
train_data['Cabin'] = words
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T<prepare_x_and_y>
train_data['Cabin'].value_counts()
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0<create_dataframe>
cabins = []
for i in test_data['Cabin']:
cabins.append(str(i))
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 md.is_multi = False
return md<find_best_params>
words = []
for i in cabins:
word = i[0]
words.append(word )
cut,lr_cut = model_meta[arch]<choose_model_class>
test_data['Cabin'] = words
def get_base() :
layers = cut_model(arch(True), cut)
 return nn.Sequential(*layers )<concatenate>
test_data['Cabin'].value_counts()
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]<compute_test_metric>
train_data['Embarked'].isna().sum()
def IoU(pred, targs):
pred =(pred > 0.5 ).astype(float)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )<compute_test_metric>
train_data['Embarked'].value_counts()
def get_score(pred, true):
n_th = 10
b = 4
thresholds = [0.5 + 0.05*i for i in range(n_th)]
n_masks = len(true)
 n_pred = len(pred)
 ious = []
score = 0
for mask in true:
buf = []
for p in pred: buf.append(IoU(p,mask))
ious.append(buf)
 for t in thresholds:
tp, fp, fn = 0, 0, 0
for i in range(n_masks):
match = False
for j in range(n_pred):
if ious[i][j] > t: match = True
if not match: fn += 1
for j in range(n_pred):
match = False
for i in range(n_masks):
if ious[i][j] > t: match = True
if match: tp += 1
else: fp += 1
score +=(( b+1)*tp)/(( b+1)*tp + b*fn + fp)
 return score/n_th<categorify>
for d in df:
d['Embarked'].fillna('S',inplace=True )
def split_mask(mask):
threshold = 0.5
threshold_obj = 8
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result<categorify>
train_data.isna().sum()
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result<compute_train_metric>
for d in df:
d['Fare'].fillna(d['Fare'].mean() ,inplace = True )
class Score_eval() :
def __init__(self):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 self.score, self.count = 0.0, 0
def put(self,pred,name):
true = get_mask_ind(name, self.segmentation_df)
 self.score += get_score(pred,true)
 self.count += 1
def evaluate(self):
return self.score/self.count<choose_model_class>
test_data.isna().sum()
m = to_gpu(Unet34(get_base()))
models = UnetModel(m )<define_variables>
train_data['Family'] = train_data.apply(lambda x: x['SibSp'] + x['Parch'], axis = 1)
 test_data['Family'] = test_data.apply(lambda x: x['SibSp'] + x['Parch'], axis = 1 )
sz = 768
bs = 8
md = get_data(sz,bs )<choose_model_class>
train_data.drop(['SibSp','Name','Ticket','Parch'], axis = 1,inplace = True)
 test_data.drop(['SibSp','Name','Ticket','Parch'], axis = 1, inplace = True )
learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH<find_best_model_class>
train_df = pd.get_dummies(train_data)
 test_df = pd.get_dummies(test_data )
def model_pred(learner, dl, F_save):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
py = to_np(F.sigmoid(learn.model(V(x))))
batch_size = len(py)
 for i in range(batch_size):
F_save(py[i],to_np(y[i]),name_list[count])
 count += 1<compute_train_metric>
train_df.drop('PassengerId', axis = 1, inplace = True )
score = Score_eval()
process_pred = lambda yp, y, name : score.put(split_mask(yp),name)
 model_pred(learn, md.val_dl, process_pred)
 print('
',score.evaluate() )<load_from_csv>
y = train_df['Survived']
train_df.drop('Survived', axis=1, inplace = True)
 train_df.drop('Cabin_T', axis=1, inplace = True)
 test_df.drop('PassengerId',axis=1, inplace=True)
 X = train_df
X_test = test_df
ship_detection = pd.read_csv(DETECTION_TEST_PRED)
 ship_detection.head()<data_type_conversions>
rfc = RandomForestClassifier()
test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()
test_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()
len(test_names), len(test_names_nothing )<set_options>
param_grid = {
'n_estimators':[200,500,1000],
'max_features':['auto'],
'max_depth': [6, 7, 8],
'criterion': ['entropy']
}
md = get_data(sz,bs)
 learn.set_data(md )<categorify>
CV = GridSearchCV(estimator = rfc, param_grid = param_grid, cv=5)
 CV.fit(X,y)
 CV.best_estimator_
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<define_variables>
rfc = RandomForestClassifier(criterion='entropy', max_depth=8, n_estimators=200)
 ada = AdaBoostClassifier()
gbc = GradientBoostingClassifier()
ship_list_dict = []
for name in test_names_nothing:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan} )<categorify>
rfc.fit(X,y)
 ada.fit(X,y)
 gbc.fit(X,y )
def enc_test(yp, y, name):
masks = split_mask(yp)
 if(len(masks)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 for mask in masks:
ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)} )<save_to_csv>
model = StackingCVClassifier(classifiers =(rfc,ada,gbc),
meta_classifier = rfc,
use_features_in_secondary = True )
model_pred(learn, md.test_dl, enc_test)
 pred_df = pd.DataFrame(ship_list_dict)
 pred_df.to_csv('submission.csv', index=False )<define_search_space>
model.fit(X.values,y )
BATCH_SIZE = 32
EDGE_CROP = 16
GAUSSIAN_NOISE = 0.1
UPSAMPLE_MODE = 'SIMPLE'
NET_SCALING =(1, 1)
 IMG_SCALING =(4, 4)
 VALID_IMG_COUNT = 600
MAX_TRAIN_STEPS = 30
AUGMENT_BRIGHTNESS = False<define_variables>
print(model.score(X, y))
montage_rgb = lambda x: np.stack([montage(x[:, :, :, i])for i in range(x.shape[3])], -1)
 ship_dir = '.. /input'
train_image_dir = os.path.join(ship_dir, 'train')
 test_image_dir = os.path.join(ship_dir, 'test')
 def multi_rle_encode(img):
labels = label(img)
 if img.ndim > 2:
return [rle_encode(np.sum(labels==k, axis=2)) for k in np.unique(labels[labels>0])]
else:
return [rle_encode(labels==k)for k in np.unique(labels[labels>0])]
def rle_encode(img):

 if np.max(img)< 1e-3:
return ''
pixels = img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle_decode(mask_rle, shape=(768, 768)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def masks_as_image(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.uint8)
 for mask in in_mask_list:
if isinstance(mask, str):
all_masks |= rle_decode(mask)
 return all_masks
def masks_as_color(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.float)
 channel = 0
for i,mask in enumerate(in_mask_list):
if isinstance(mask, str):
all_masks[:,:] += np.log1p(i+1)* rle_decode(mask)
 return all_masks<load_from_csv>
prediction = model.predict(X_test.values )
<categorify><EOS> 
output = pd.DataFrame({'PassengerId' : test_data.PassengerId, 'Survived' : prediction})
 output.to_csv('my_submissions.csv', index = False )