Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
warnings.filterwarnings('ignore' )
outs = [pd.read_csv(f, index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "m" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True )<feature_engineering>
df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv' )
rank = np.tril(concat_sub.iloc[:,1:].corr().values,-1)
 m =(rank>0 ).sum()
m_gmean, s = 0, 0
for n in range(min(rank.shape[0],m)) :
mx = np.unravel_index(rank.argmin() , rank.shape)
 w =(m-n)/(m+n/10)
 print(w)
 m_gmean += w*(np.log(concat_sub.iloc[:,mx[0]+1])+np.log(concat_sub.iloc[:,mx[1]+1])) /2
s += w
rank[mx] = 1
m_gmean = np.exp(m_gmean/s ).clip(0.0,1.0 )<load_from_csv>
df_test['Survived'] = 999
predict_list = []
predict_list.append(pd.read_csv(".. /input/cellstack/submission-174.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/cellstack/submission-201.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/cellstack/submission-231.csv")[LABELS].values )<save_to_csv>
df = pd.concat([df_train , df_test] , axis = 0)
 print(df_train.shape)
 print(df_test.shape)
 print('Combined dataframe shape :',df.shape )
warnings.filterwarnings("ignore")
 print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions = predictions /len(predict_list)
 submission = pd.read_csv('.. /input/recursion-cellular-image-classification/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('AggStacker.csv', index=False )<define_variables>
print('The number of null values in age columns',df['Age'].isnull().sum())
 print('The % of null values in age columns',round(df['Age'].isnull().mean() * 100,2))
sub_path = ".. /input/cellstack"
all_files = os.listdir(sub_path)
 all_files<feature_engineering>
print('The number of null values in cabin columns',df['Cabin'].isnull().sum())
 print('The % of null values in cabin columns',round(df['Cabin'].isnull().mean() * 100,2))
concat_sub['m_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['m_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['m_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
df.drop(columns = 'Cabin' , inplace = True )
cutoff_lo = 0.8
cutoff_hi = 0.2<save_to_csv>
print('The number of null values in Embarked is ', df['Embarked'].isnull().sum() )
concat_sub['sirna'] = m_gmean.astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )<save_to_csv>
df['Embarked'].replace({np.nan:'S'} , inplace = True )
concat_sub['sirna'] = concat_sub['m_median'].astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
df[df['Fare'].isnull() ]
concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['m_median']))
concat_sub[['id_code','sirna']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )<feature_engineering>
df['Name'].isnull().sum()
concat_sub['m_mean'] = m_gmean.astype(int)
 concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_mean'])).astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )<feature_engineering>
def GetTitle_temp(name):
fname_title = name.split(',')[1]
title = fname_title.split('.')[0]
title = title.strip().lower()
return title
df.Name.map(GetTitle_temp ).value_counts()
concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_median'])).astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )<load_from_csv>
df['Parch'].value_counts()
SIZE = 224
NUM_CLASSES = 1108
train_csv = pd.read_csv(".. /input/recursion-cellular-image-classification/train.csv")
 test_csv = pd.read_csv(".. /input/recursion-cellular-image-classification/test.csv")
 sub = pd.read_csv(".. /input/recursion-cellular-keras-densenet/submission.csv" )<concatenate>
df['SibSp'].value_counts()
np.stack([train_csv.plate.values[train_csv.sirna == i] for i in range(10)] ).transpose()<count_values>
df['Accomp'] = df['SibSp'] + df['Parch']
df.drop(columns = ['SibSp' , 'Parch'] , inplace = True )
train_csv.loc[train_csv.sirna==0,'plate'].value_counts()<count_values>
df['Sex'].value_counts()
plate_groups = np.zeros(( 1108,4), int)
 for sirna in range(1108):
grp = train_csv.loc[train_csv.sirna==sirna,:].plate.value_counts().index.values
assert len(grp)== 3
plate_groups[sirna,0:3] = grp
plate_groups[sirna,3] = 10 - grp.sum()
plate_groups[:10,:]<feature_engineering>
df['Sex'] = df['Sex'].map({'female':0 , 'male':1 } )
all_test_exp = test_csv.experiment.unique()
group_plate_probs = np.zeros(( len(all_test_exp),4))
for idx in range(len(all_test_exp)) :
preds = sub.loc[test_csv.experiment == all_test_exp[idx],'sirna'].values
pp_mult = np.zeros(( len(preds),1108))
pp_mult[range(len(preds)) ,preds] = 1
sub_test = test_csv.loc[test_csv.experiment == all_test_exp[idx],:]
assert len(pp_mult)== len(sub_test)
 for j in range(4):
mask = np.repeat(plate_groups[np.newaxis, :, j], len(pp_mult), axis=0)== \
np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)
 group_plate_probs[idx,j] = np.array(pp_mult)[mask].sum() /len(pp_mult )<create_dataframe>
df.drop(columns = ['Ticket'] , inplace = True )
pd.DataFrame(group_plate_probs, index = all_test_exp )<groupby>
cat = pd.get_dummies(df[['Embarked' , 'Name']] , drop_first=True )
exp_to_group = group_plate_probs.argmax(1)
 print(exp_to_group )<choose_model_class>
df = pd.concat([df,cat] , axis = 1)
 df.drop(columns = ['Embarked' , 'Name'] , inplace = True )
def create_model(input_shape,n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dense(1024, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<choose_model_class>
df.isnull().mean() *100
model = create_model(input_shape=(SIZE,SIZE,3),n_out=NUM_CLASSES )<load_pretrained>
knn_imputer = KNN()
df_knn = df.copy()
df_knn.iloc[:,:] = knn_imputer.fit_transform(df_knn )
model.load_weights('.. /input/recursion-cellular-keras-densenet/Densenet121.h5' )<predict_on_test>
MICE_imputer = IterativeImputer()
df_mice = df.copy()
df_mice.iloc[:,:] = knn_imputer.fit_transform(df_mice )
predicted = []
for i, name in tqdm(enumerate(test_csv['id_code'])) :
path1 = os.path.join('.. /input/recursion-cellular-image-classification-224-jpg/test/test/', name+'_s1.jpeg')
 image1 = cv2.imread(path1)
 score_predict1 = model.predict(( image1[np.newaxis])/255)
 path2 = os.path.join('.. /input/recursion-cellular-image-classification-224-jpg/test/test/', name+'_s2.jpeg')
 image2 = cv2.imread(path2)
 score_predict2 = model.predict(( image2[np.newaxis])/255)
 predicted.append(0.5*(score_predict1 + score_predict2)) 
<feature_engineering>
X, y = dfm_train.drop(columns = ['Survived','PassengerId']), dfm_train['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, random_state=123)
 xg_cl = xgb.XGBClassifier(objective='binary:logistic',
n_estimators=20, seed=123)
 xg_cl.fit(X_train, y_train)
 preds = xg_cl.predict(X_test)
 accuracy = float(np.sum(preds==y_test)) /y_test.shape[0]
print("accuracy: %f" %(accuracy))
def select_plate_group(pp_mult, idx):
sub_test = test_csv.loc[test_csv.experiment == all_test_exp[idx],:]
assert len(pp_mult)== len(sub_test)
 mask = np.repeat(plate_groups[np.newaxis, :, exp_to_group[idx]], len(pp_mult), axis=0)!= \
np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)
 pp_mult[mask] = 0
return pp_mult<feature_engineering>
confusion_matrix(y_test , preds )
for idx in range(len(all_test_exp)) :
indices =(test_csv.experiment == all_test_exp[idx])
 preds = predicted[indices,:].copy()
preds = select_plate_group(preds, idx)
 sub.loc[indices,'sirna'] = preds.argmax(1 )<load_from_csv>
y_pred = xg_cl.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_pred})
 results.to_csv('Titanic Prediction_XGB.csv', index=False )
( sub.sirna == pd.read_csv(".. /input/recursion-cellular-keras-densenet/submission.csv" ).sirna ).mean()<save_to_csv>
X, y = dfm_train.drop(columns = ['Survived','PassengerId']), dfm_train['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, random_state=123)
 params = {
'min_child_weight': [1, 3,5,7 , 10],
'gamma': [0.5, 1, 1.5, 2,3,4, 5],
'subsample': [0.6,0.7, 0.8,0.9, 1.0],
'colsample_bytree': [0.6,0.7, 0.8,0.9, 1.0],
'max_depth': [3, 4, 5 , 6]
}
xgb = xgb.XGBClassifier(learning_rate=0.02, n_estimators=600, objective='binary:logistic',
silent=True, nthread=1)
 folds = 3
param_comb = 5
skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 1001)
 random_search = RandomizedSearchCV(xgb, param_distributions=params, n_iter=param_comb, scoring='roc_auc', n_jobs=4, cv=skf.split(X,y), verbose=3, random_state=1001)
 random_search.fit(X, y)
sub.to_csv('.. /working/submission.csv', index=False, columns=['id_code','sirna'] )<feature_engineering>
random_search.best_params_
os.environ['CUDA_LAUNCH_BLOCKING'] = '1' 
<set_options>
y_test = random_search.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic Prediction_XGB_hp.csv', index=False)
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 0
seed_everything(SEED )<load_from_csv>
X, y = dfm_train.drop(columns = ['Survived','PassengerId']), dfm_train['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, random_state=123)
 params = {
'min_child_weight': [1, 3,5,7 , 10],
'gamma': [0.5, 1, 1.5, 2,3,4, 5],
'subsample': [0.6,0.7, 0.8,0.9, 1.0],
'colsample_bytree': [0.6,0.7, 0.8,0.9, 1.0],
'max_depth': [3, 4, 5 , 6]
}
xgb = xgb.XGBClassifier(learning_rate=0.02, n_estimators=600, objective='binary:logistic',
silent=True, nthread=1)
 folds = 3
param_comb = 5
skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 1001)
 random_search = RandomizedSearchCV(xgb, param_distributions=params, n_iter=param_comb, scoring='roc_auc', n_jobs=4, cv=skf.split(X,y), verbose=3, random_state=1001)
 random_search.fit(X, y)
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head(10 )<categorify>
y_test = random_search.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic Predictionkn.csv', index=False)
def generate_df(train_df,sample_num=1):
train_df['path'] = train_df['experiment'].str.cat(train_df['plate'].astype(str ).str.cat(train_df['well'],sep='/'),sep='/Plate')+ '_s'+str(sample_num)+ '_w'
train_df = train_df.drop(columns=['id_code','experiment','plate','well'] ).reindex(columns=['path','sirna'])
 return train_df
proc_train_df = generate_df(train_df )<load_pretrained>
knn = KNeighborsClassifier()
params = {
'n_neighbors' : sp_randint(1 , 20),
'p' : sp_randint(1 , 5),
}
rsearch_knn = RandomizedSearchCV(knn , param_distributions = params , cv = 3 , random_state= 3 , n_jobs = -1 , return_train_score=True)
 rsearch_knn.fit(X , y )
il = MultiChannelImageList.from_df(df=proc_train_df,path='.. /input/train/' )<categorify>
rsearch_knn.best_params_
def image2np(image:Tensor)->np.ndarray:
"Convert from torch style `image` to numpy/matplotlib style."
res = image.cpu().permute(1,2,0 ).numpy()
if res.shape[2]==1:
return res[...,0]
elif res.shape[2]>3:
return res[...,:3]
else:
return res
vision.image.image2np = image2np<split>
rfc = RandomForestClassifier(random_state=3)
 params = { 'n_estimators' : sp_randint(50 , 200),
'max_features' : sp_randint(1 , 12),
'max_depth' : sp_randint(2,10),
'min_samples_split' : sp_randint(2,20),
'min_samples_leaf' : sp_randint(1,20),
'criterion' : ['gini' , 'entropy']
}
rsearch_rfc = RandomizedSearchCV(rfc , param_distributions= params , n_iter= 200 , cv = 3 , scoring='roc_auc' , random_state= 3 , return_train_score=True , n_jobs=-1)
 rsearch_rfc.fit(X,y )
train_df,val_df = train_test_split(proc_train_df,test_size=0.035, stratify = proc_train_df.sirna, random_state=42)
 _proc_train_df = pd.concat([train_df,val_df] )<categorify>
rsearch_rfc.best_params_
data =(MultiChannelImageList.from_df(df=_proc_train_df,path='.. /input/train/')
.split_by_idx(list(range(len(train_df),len(_proc_train_df)))) 
.label_from_df() 
.transform(get_transforms() ,size=256)
.databunch(bs=128,num_workers=4)
.normalize()
)<define_variables>
lr = LogisticRegression(solver = 'liblinear')
 knn = KNeighborsClassifier(**rsearch_knn.best_params_)
 rfc = RandomForestClassifier(**rsearch_rfc.best_params_)
 clf = VotingClassifier(estimators=[('lr' ,lr),('knn' , knn),('rfc' , rfc)] , voting = 'soft')
 clf.fit(X , y )
data.show_batch()<install_modules>
y_test = clf.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic Prediction_Stack.csv', index=False)
!pip install efficientnet_pytorch<import_modules>
X, y = dfm_train.drop(columns = ['Survived','PassengerId']), dfm_train['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, random_state=123 )
from efficientnet_pytorch import *<choose_model_class>
RFM = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=123,
n_jobs=-1,
verbose=1)
 RFM.fit(X,y )
RESNET_MODELS = {
18: torchvision.models.resnet18,
34: torchvision.models.resnet34,
50: torchvision.models.resnet50,
101: torchvision.models.resnet101,
152: torchvision.models.resnet152,
}
def resnet_multichannel(depth=50,pretrained=True,num_classes=1108,num_channels=6):
model = RESNET_MODELS[depth](pretrained=pretrained)
 w = model.conv1.weight
model.conv1 = nn.Conv2d(num_channels, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 model.conv1.weight = nn.Parameter(torch.stack([torch.mean(w, 1)]*num_channels, dim=1))
return model
DENSENET_MODELS = {
121: torchvision.models.densenet121,
161: torchvision.models.densenet161,
169: torchvision.models.densenet169,
201: torchvision.models.densenet201,
}
def densenet_multichannel(depth=121,pretrained=True,num_classes=1108,num_channels=6):
model = DENSENET_MODELS[depth](pretrained=pretrained)
 w = model.features.conv0.weight
model.features.conv0 = nn.Conv2d(num_channels, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 model.features.conv0.weight = nn.Parameter(torch.stack([torch.mean(w, 1)]*num_channels, dim=1))
return model
def efficientnet_multichannel(pretrained=True,name='b0',num_classes=1108,num_channels=6,image_size=256):
model = EfficientNet.from_pretrained('efficientnet-'+name,num_classes=num_classes)
 w = model._conv_stem.weight
model._conv_stem = utils.Conv2dStaticSamePadding(num_channels,32,kernel_size=(3, 3), stride=(2, 2), bias=False, image_size = image_size)
 model._conv_stem.weight = nn.Parameter(torch.stack([torch.mean(w, 1)]*num_channels, dim=1))
return model<define_search_model>
y_pred = RFM.predict(X_test )
def resnet18(pretrained,num_channels=6):
return resnet_multichannel(depth=18,pretrained=pretrained,num_channels=num_channels)
 def _resnet_split(m): return(m[0][6],m[1])
 def densenet161(pretrained,num_channels=6):
return densenet_multichannel(depth=161,pretrained=pretrained,num_channels=num_channels)
 def _densenet_split(m:nn.Module): return(m[0][0][7],m[1])
 def efficientnetb0(pretrained=True,num_channels=6):
return efficientnet_multichannel(pretrained=pretrained,name='b0',num_channels=num_channels)
<choose_model_class>
y_test = RFM.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic PredictionRFM.csv', index=False)
learn = Learner(data, efficientnetb0() ,metrics=[accuracy] ).to_fp16()
learn.path = Path('.. /' )<train_model>
X, y = dfm_train.drop(columns = ['Survived','PassengerId']), dfm_train['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, random_state=123 )
learn.unfreeze() 
<train_model>
clf_ET = ExtraTreesClassifier(random_state=0, bootstrap=True, oob_score=True)
 sss = model_selection.StratifiedShuffleSplit(n_splits=10, test_size=0.33, random_state= 0)
 sss.get_n_splits(X, y)
 parameters = {'n_estimators' : np.r_[10:210:10],
'max_depth': np.r_[1:6]
}
grid = model_selection.GridSearchCV(clf_ET, param_grid=parameters, scoring = 'accuracy', cv = sss, return_train_score=True, n_jobs=4, verbose=2)
 grid.fit(X,y )
learn.fit_one_cycle(18,1e-3 )<load_from_csv>
y_test = RFM.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic PredictionETC.csv', index=False )