Kaggle competition: 2019  ML competition with KISTI
<SOS> metric: accuracy Kaggle data source: 2019-ml-competition-with-kisti<define_variables>
import graphviz
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn import tree
from sklearn.model_selection import cross_val_score, GridSearchCV
tr_sample = train.drop('label', axis=1 ).values.reshape(-1,28,28)[0]
ts_sample = test.values.reshape(-1,28,28)[0]<count_values>
train = pd.read_csv('.. /input/train.csv' ).set_index('PassengerId')
 test = pd.read_csv('.. /input/test.csv' ).set_index('PassengerId')
 df = pd.concat([train, test], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrChild'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['FamilyTotalCount'] = family.transform(lambda s: s[df.IsWomanOrChild].fillna(0 ).count())
 df['FamilyTotalCount'] = df.mask(df.IsWomanOrChild, df.FamilyTotalCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrChild].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrChild, df.FamilySurvivedCount - df.Survived.fillna(0), axis=0)
 df['FamilySurvivalRate'] =(df.FamilySurvivedCount / df.FamilyTotalCount.replace(0, np.nan))
df['IsSingleTraveler'] = df.FamilyTotalCount == 0
test['pixel345'].value_counts()<import_modules>
name = df.groupby(df.LastName ).Ticket
name.value_counts()
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split<split>
df['SingleTraveler'] = 3
ticket_list_single = []
ticket_list_family = []
ticket_list_null = []
for ticket_id in list(df['Ticket'].unique()):
count = df[df['Ticket']==ticket_id].count() [0]
if count > 1 :
ticket_list_family.append(ticket_id)
 else:
ticket_list_single.append(ticket_id)
 def tune_SingleTraveler(df):
for ticket in ticket_list_single:
df.ix[df.Ticket == ticket, "SingleTraveler"] = True
for ticket in ticket_list_family:
df.ix[df.Ticket == ticket, "SingleTraveler"] = False
tune_SingleTraveler(df)
x_test = test.values.reshape(-1,28,28,1)
 x_test = x_test/255
x_train_full = train.drop('label', axis=1 ).values.reshape(-1,28,28,1)
 y_train_full = train['label'].values
x_train, x_val, y_train, y_val = train_test_split(x_train_full, y_train_full, test_size=0.2, random_state=42 )<train_model>
names = []
for name in list(df['LastName'].unique()):
count = df[df['LastName']==name].count() [0]
if count > 1 :
names.append(name)
 for name in names:
df.ix[df.LastName == name, "SingleTraveler"] = False
print(df['SingleTraveler'].sum() )
batch_size=64
img_gen = ImageDataGenerator(rescale=1/255,
rotation_range=30,
zoom_range=.1,
shear_range=.1,
width_shift_range=.25,
height_shift_range=.25)
 train_gen = img_gen.flow(x_train, y_train,
batch_size=batch_size)
 valid_gen = img_gen.flow(x_val, y_val,
batch_size=batch_size,
shuffle=False )<import_modules>
df['Family_freq'] = df.groupby('LastName')['LastName'].transform('count' )
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation, Dense, Conv2D, MaxPool2D, Dropout, BatchNormalization, AveragePooling2D, GlobalAveragePooling2D
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard<choose_model_class>
df['Sex'].isnull().sum()
early_stop = EarlyStopping(monitor='val_loss', patience=5, mode='min',restore_best_weights=True)
 check_point = ModelCheckpoint('digit_reg_mnist_z.h5', monitor='val_accuracy', save_best_only=True)
 lr_plateau = ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
factor=.2,
min_lr=1e-6 )<choose_model_class>
df['All'] =(( df.Sex == 'male')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['FamilyTotalCountM'] = family.transform(lambda s: s[df.All].fillna(0 ).count())
 df['FamilyTotalCountM'] = df.mask(df.All, df.FamilyTotalCountM - 1, axis=0)
 df['FamilySurvivedCountM'] = family.transform(lambda s: s[df.All].fillna(0 ).sum())
 df['FamilySurvivedCountM'] = df.mask(df.All, df.FamilySurvivedCountM - df.Survived.fillna(0), axis=0)
 df['FamilySurvivalRateM'] =(df.FamilySurvivedCountM / df.FamilyTotalCountM.replace(0, np.nan))
model = Sequential()
model.add(Conv2D(32, kernel_size=(3,3), input_shape=(28,28,1), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(Conv2D(64, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(64, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(Conv2D(128, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(AveragePooling2D(pool_size=(2,2)))
 model.add(Conv2D(128, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(Conv2D(128, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(256, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(Conv2D(256, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization(momentum=.9, epsilon=1e-5))
model.add(Activation('relu'))
model.add(GlobalAveragePooling2D())
 model.add(Dense(10, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adamax',
metrics=['accuracy'])
 model.summary()<train_model>
df['All'].isnull().sum()
model.fit(train_gen,
epochs=100,
steps_per_epoch=250,
validation_data=valid_gen,
callbacks=[lr_plateau, early_stop] )<import_modules>
_ = df.rename({'Cabin':'Deck'},axis=1,inplace=True)
 df['Deck'] = df['Deck'].fillna('N')
 len(df['Deck'] )
import seaborn as sns
from sklearn.metrics import classification_report, confusion_matrix<create_dataframe>
def cabin_to_deck(row):
return row['Deck'][0]
df['Deck'] = df.apply(cabin_to_deck,axis=1 )
eval_df = pd.DataFrame(model.history.history)
 length = len(eval_df )<predict_on_test>
ticket_list = []
for ticket_id in list(df['Ticket'].unique()):
count = df[df['Ticket']==ticket_id].count() [0]
decks = df[df['Ticket']==ticket_id]['Deck']
empty_decks =(decks=='N' ).sum()
if(count > 1)and(empty_decks > 0)and(empty_decks < len(decks)) :
ticket_list.append(ticket_id)
 print(ticket_list )
pred = np.argmax(model.predict(valid_gen), axis=1)
 pred<compute_test_metric>
for ticket in ticket_list:
display(df[df['Ticket']==ticket] )
print(classification_report(y_val, pred))<train_model>
_ = df.set_value(533,'Deck',value=df.loc[128]['Deck'])
 _ = df.set_value(1308,'Deck',value=df.loc[128]['Deck'])
 _ = df.set_value(258,'Deck',df.loc[679]['Deck'])
 _ = df.set_value(373,'Deck',value='C')
 _ = df.set_value(290,'Deck',value=df.loc[741]['Deck'])
 _ = df.set_value(708,'Deck',value=df.loc[297]['Deck'])
 _ = df.set_value(1032,'Deck',value=df.loc[297]['Deck'])
 _ = df.set_value(306,'Deck',value='C')
 _ = df.set_value(1266,'Deck',value=df.loc[1033]['Deck'])
 _ = df.set_value(856,'Deck',value=df.loc[318]['Deck'])
 _ = df.set_value(1108,'Deck',value=df.loc[318]['Deck'])
 _ = df.set_value(380,'Deck',value='C')
 _ = df.set_value(557,'Deck',value='C')
 _ = df.set_value(537,'Deck',value='C')
 _ = df.set_value(1215,'Deck',value='E')
 _ = df.set_value(841,'Deck',value=df.loc[772]['Deck'])
lr_plateau = ReduceLROnPlateau(monitor='accuracy',
patience=2,
factor=.2,
min_lr=1e-6)
 full_train_gen = img_gen.flow(x_train_full, y_train_full,
batch_size=batch_size )<train_model>
for i in range(3):
if 'N' in decks_by_class[i]:
decks_by_class[i].remove('N')
 if 'T' in decks_by_class[i]:
decks_by_class[i].remove('T' )
model.fit(full_train_gen,
epochs=22,
steps_per_epoch=250,
callbacks=[lr_plateau] )<predict_on_test>
weights_by_class = [[],[],[]]
for i,deck_list in enumerate(decks_by_class):
for deck in deck_list:
if i == 0:
class_total = df[(df['Deck']!='N')&(df['Pclass']==i+1)].count() [0]-1
else:
class_total = df[(df['Deck']!='N')&(df['Pclass']==i+1)].count() [0]
deck_total = df[(df['Deck']==deck)&(df['Pclass']==i+1)].count() [0]
weights_by_class[i].append(deck_total/class_total)
 print(f'Pclass = {i+1} weights:',np.round(weights_by_class[i],3))
real_pred = np.argmax(model.predict(x_test), axis=1 )<save_to_csv>
ticket_dict = {}
submit_df = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submit_df['Label'] = real_pred
submit_df.to_csv('submission.csv', index=False )<set_options>
def impute_deck(row):
ticket = row['Ticket']
deck = row['Deck']
pclass = row['Pclass']
if(deck == 'N')and(ticket not in ticket_dict):
if pclass == 1:
deck = list(np.random.choice(decks_by_class[0],size=1,
p=weights_by_class[0])) [0]
elif pclass ==2:
deck = list(np.random.choice(decks_by_class[1],size=1,
p=weights_by_class[1])) [0]
elif pclass ==3:
deck = list(np.random.choice(decks_by_class[2],size=1,
p=weights_by_class[2])) [0]
ticket_dict[ticket] = deck
elif(deck == 'N')and(ticket in ticket_dict):
deck = ticket_dict[ticket]
return deck
%matplotlib inline
np.random.seed(0)
 sns.set_palette('pastel' )<load_from_csv>
df['Deck'] = df.apply(impute_deck,axis=1 )
train = pd.read_csv(r'/kaggle/input/titanic/train.csv')
 test = pd.read_csv(r'/kaggle/input/titanic/test.csv')
 train.tail()<count_missing_values>
df['Deck'] = df['Deck'].map({'F':0,'C':1,'E':2,'G':3,'D':4,'A':5,
'B':6,'T':7} ).astype(int )
train.isnull().sum()<count_missing_values>
train.isnull().sum()<count_missing_values>
df.ix[df.Title == 'Master', "Sex"] = 'boy'
test.isnull().sum()<count_missing_values>
x = pd.concat([
df.FamilySurvivalRate.fillna(0),
df.SingleTraveler,
df.Sex.replace({'male': 0, 'female': 1, 'boy': 2}),
df.Deck,
], axis=1)
 train_x, test_x = x.loc[train.index], x.loc[test.index]
train_y = df.Survived.loc[train.index]
test.isnull().sum()<groupby>
clf_dt = tree.DecisionTreeClassifier()
grid = GridSearchCV(clf_dt, cv=5, param_grid={
'criterion': ['gini', 'entropy'],
'max_depth': [2, 3, 4, 5]})
 grid.fit(train_x, train_y)
 grid.best_params_
avg_age_train =(train.groupby("Sex")['Age'] ).mean()
print(avg_age_train)
 avg_age_test =(test.groupby("Sex")['Age'] ).mean()
print(avg_age_test )<feature_engineering>
model_dt = grid.best_estimator_
for i in range(len(train['Age'])) :
if train['Age'].isnull() [i] == True and train['Sex'][i] == 'male':
train['Age'][i] = np.round(avg_age_train['male'],decimals=1)
 elif train['Age'].isnull() [i] == True and train['Sex'][i] == 'female':
train['Age'][i] = np.round(avg_age_train['female'],decimals=1)
 for i in range(len(test['Age'])) :
if test['Age'].isnull() [i] == True and test['Sex'][i] == 'male':
test['Age'][i] = np.round(avg_age_test['male'],decimals=1)
 elif test['Age'].isnull() [i] == True and test['Sex'][i] == 'female':
test['Age'][i] = np.round(avg_age_test['female'],decimals=1 )<drop_column>
model_dt.score(train_x, train_y )
train = train.reset_index(drop=True)
 test = test.reset_index(drop=True )<count_missing_values>
test_y = model_dt.predict(test_x ).astype(int)
 pd.DataFrame({'Survived': test_y}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_dt.csv', index=False )
train.isnull().sum()<count_missing_values>
preds = pd.DataFrame()
preds = x.loc[test.index]
preds['Pclass'] = test['Pclass']
preds['pred'] = model_dt.predict_proba(test_x)[:, 1]
preds = preds.drop(['FamilySurvivalRate','SingleTraveler','Sex', 'Deck'], axis=1)
 preds.head()
train.isnull().sum()<count_missing_values>
preds.groupby(['Pclass'] ).pred.value_counts()
test.isnull().sum()<count_missing_values>
preds.loc[preds['Pclass'] == 3, 'pred'] = preds.loc[preds['Pclass'] == 3, 'pred'] -0.3
sub = []
sub = preds['pred'].values
sub = np.around(sub ).astype(int )
test.isnull().sum()<prepare_x_and_y>
test_y = sub.astype(int)
 pd.DataFrame({'Survived': test_y}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_dt_m0.3.csv', index=False )
Y = train['Survived']
train = train.drop('Survived',axis=1)
 data = pd.concat([train,test],axis=0)
 data.head()<groupby>
model_xgb = xgboost.XGBClassifier()
model_xgb.fit(train_x, train_y)
 test_y = model_xgb.predict(test_x ).astype(int)
 pd.DataFrame({'Survived': test_y}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_xgb.csv', index=False )
avg_fare = data.groupby("Sex")['Fare'].mean()
avg_fare<filter>
X_train = train_x
Y_train = train_y
print("Index of the null value is: ",test[test['Fare'].isnull() ].index.tolist())
 print(test['Sex'][152] )<feature_engineering>
def crossValidation(data,target, model, cv = 5):
ksplits = KFold(n_splits=cv)
 errors = []
for j,(indexTrain, indexTest)in enumerate(ksplits.split(data, target)) :
print("This is fold ", j + 1, "of the cross validation")
 x_train, y_train = data.iloc[indexTrain, :], target.iloc[indexTrain]
x_test, y_test = data.iloc[indexTest, :], target.iloc[indexTest]
print("Fitting the model")
 model.fit(x_train, y_train)
 predictions = model.predict(x_test)
 errorFold = accuracy_score(y_test, predictions)
 errors.append(errorFold)
 print("The mean accuracy score over the folds is: ", np.mean(errors))
return
data['Fare'][152] = avg_fare['male']<count_missing_values>
modelXGboost_tuned = xgboost.XGBClassifier(base_score = 0.5,
colsample_bytree= 0.65,
gamma= 0,
learning_rate= 0.5,
max_depth= 3,
min_child_weight= 1,
n_estimators= 120,
scale_pos_weight= 1)
 crossValidation(X_train, Y_train, modelXGboost_tuned )
data.isnull().sum()<count_missing_values>
modelK_tuned = KNeighborsClassifier(algorithm='ball_tree', n_neighbors=5, p=1, weights='distance')
 crossValidation(X_train, Y_train, modelK_tuned )
data.isnull().sum()<count_duplicates>
modelForest_tuned = RandomForestClassifier(max_depth =5, n_estimators=25)
 crossValidation(X_train, Y_train, modelForest_tuned )
print("Number of duplicate rows in the train dataset :",train.duplicated().sum())
 print("Number of duplicate rows in the test dataset :",test.duplicated().sum() )<feature_engineering>
modelLogistic = LogisticRegression(solver="liblinear")
 crossValidation(X_train, Y_train, modelLogistic )
Name_title_data = data['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 print(Name_title_data)
 data['Name_title'] = Name_title_data
data = data.reset_index(drop=True)
 data.head()<feature_engineering>
modelXGboost = xgboost.XGBClassifier()
crossValidation(X_train, Y_train, modelXGboost )
age_group_data = [None] * len(data['Age'])
 for i in range(len(data['Age'])) :
if data['Age'][i] <= 3:
age_group_data[i] = 'Baby'
elif data['Age'][i] >3 and data['Age'][i] <= 13:
age_group_data[i] = 'Child'
elif data['Age'][i] >13 and data['Age'][i] <= 19:
age_group_data[i] = 'Teenager'
elif data['Age'][i] >19 and data['Age'][i] <= 30:
age_group_data[i] = 'Young Adult'
elif data['Age'][i] >30 and data['Age'][i] <= 45:
age_group_data[i] = 'Middle Aged Adult'
elif data['Age'][i] >45 and data['Age'][i] <65:
age_group_data[i] = 'Adult'
else:
age_group_data[i] = 'Old'
data['age_group'] = age_group_data<feature_engineering>
modelDiscriminant = LinearDiscriminantAnalysis()
crossValidation(X_train, Y_train, modelDiscriminant )
data['Is_Married'] = 0
data['Is_Married'].loc[data['Name_title'] == 'Mrs'] = 1
data['FamSize'] = data['SibSp'] + data['Parch'] + 1
data['Single'] = data['FamSize'].map(lambda s: 1 if s == 1 else 0 )<groupby>
modelForest = RandomForestClassifier()
crossValidation(X_train, Y_train, modelForest )
np.unique(data['Ticket'])
 tic = data.groupby('Ticket',sort=True,group_keys=True)
 groups = list(tic.groups)
 togther = [None] * len(data['Ticket'])
 k=0
for i in range(len(groups)) :
for j in range(len(data['Ticket'])) :
if data['Ticket'][j] == groups[i]:
togther[j] = i
data['Togther'] = togther<feature_engineering>
modelK = KNeighborsClassifier()
crossValidation(X_train, Y_train, modelK )
rates = [None]*len(data['Fare'])
 for i in range(len(data['Fare'])) :
if data['Fare'][i]<=10:
rates[i] = 1
elif data['Fare'][i] >10 and data['Fare'][i]<=30:
rates[i] = 2
elif data['Fare'][i] >30 and data['Fare'][i]<=70:
rates[i] = 3
elif data['Fare'][i] >70 and data['Fare'][i]<=100:
rates[i] = 4
else:
rates[i] = 5
data['Rates'] = rates<feature_engineering>
crossValidation(X_train, Y_train, model_dt )
data['Cabin_present'] = 1
data['Cabin_present'].loc[data['Cabin'].isnull() ] = 0<drop_column>
def crossValMixed(X, y, model,cv = 5):
n_folds = KFold(n_splits = cv,shuffle=True)
 counter = 0
errorMetr = []
for indexTrain, indexTest in n_folds.split(X,y):
print("This is fold: ", counter, "of the cross validation")
 X_train, y_train = X.iloc[indexTrain, :], y.iloc[indexTrain]
X_test, y_test = X.iloc[indexTest, :], y.iloc[indexTest]
print("Fitting the model")
 predictions = model.stackingActive(X_train, y_train, X_test)
 print("The metric in the fold ", counter, "is: ", accuracy_score(y_test,predictions))
counter += 1
errorMetr.append(accuracy_score(y_test,predictions))
print("The mean absolute error over the ", cv, "folds is: ", np.mean(errorMetr))
return predictions
data = data.drop('Cabin',axis=1)
 data = data.drop('Ticket',axis=1)
 data = data.drop('Name',axis=1)
 data = data.drop('PassengerId',axis=1 )<categorify>
pd.DataFrame({'Survived': predictions.astype(int)}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_stack_rkd.csv', index=False )
data_ohe = pd.get_dummies(data,drop_first=True)
 data_ohe.head()<feature_engineering>
pd.DataFrame({'Survived': predictions.astype(int)}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_stack_xkx_tuned.csv', index=False )
train['Survived'] = Y<drop_column>
preds = pd.DataFrame()
preds = x.loc[test.index]
preds['Pclass'] = test['Pclass']
preds['pred'] = predictions_proba[:, 1]
preds.head()
train = train.drop('Survived',axis=1 )<split>
preds.loc[preds['Pclass'] == 3, 'pred'] = preds.loc[preds['Pclass'] == 3, 'pred'] -0.2
sub = []
sub = preds['pred'].values
sub = np.around(sub ).astype(int )
<split><EOS> 
pd.DataFrame({'Survived': sub}, index=test.index)\ 
.reset_index() \ 
.to_csv(f'submission_stack_xkx_m.csv', index=False )