Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
warnings.filterwarnings('ignore')
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model 
<string_transform>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
full = train.append(test,ignore_index=True)
 print('合并后的数据集：',full.shape )
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
%matplotlib inline
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
df_num = pd.DataFrame(train,columns=['Age','Fare','SibSp','Parch','Survived'])
 df_num.hist(figsize=(25,20),bins=20,xlabelsize=15,ylabelsize=15 )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
df_Cabin = train['Cabin'].astype(str ).map(lambda x:x[0])
 df_Cabin.head()
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25)
<load_from_csv>
train['Ticket'].value_counts()
def read_and_concat_dataset(training_path, test_path):
train = pd.read_csv(training_path)
 train['train'] = 1
test = pd.read_csv(test_path)
 test['train'] = 0
data = train.append(test, ignore_index=True)
 return train, test, data
train, test, data = read_and_concat_dataset('/kaggle/input/titanic/train.csv', '/kaggle/input/titanic/test.csv')
 data = data.set_index('PassengerId')
 data.info()<groupby>
def getTicket(ticket):
ticket = ticket.replace('.',' ')
 ticket = ticket.replace('/',' ')
 ticket = ticket.split(' ')
 ticket = map(lambda a:a.strip() ,ticket)
 ticket = list(filter(lambda x:not x.isdigit() ,ticket))
if len(ticket)>0:
return ticket[0]
else:
return 'Unknown'
df_tickets = train['Ticket'].map(getTicket)
 df_tickets.value_counts()
def counting_values(data, variable1, variable2):
return data[[variable1, variable2]][data[variable2].isnull() ==False].groupby([variable1], as_index=False ).mean().sort_values(by=variable2, ascending=False )<count_values>
def get_agePeriod(age):
if not np.bool(age):
return age
x = np.linspace(0,80,9)
 for i in range(0,len(x)-1):
if(age>x[i] and age<=x[i+1]):
return '(%.0f,%.0f]' %(x[i],x[i+1])
 df_num['Age'] = df_num['Age'].map(get_agePeriod)
 df_num['Age'].head()
counting_values(data, 'Sex','Survived' )<feature_engineering>
train['Survived'].value_counts()
data['Women'] = np.where(data.Sex=='female',1,0)
 comparing(data, 'Women','Survived' )<count_missing_values>
full['Age'] = full['Age'].fillna(full['Age'].median())
 full['Fare'] = full['Fare'].fillna(full['Age'].median() )
data.isnull().sum()<count_missing_values>

 pd.value_counts(full['Embarked'] )
data.isnull().sum()<groupby>
full['Embarked'] = full['Embarked'].fillna('S' )
data.groupby('Pclass' ).Fare.mean()
data.Fare = data.Fare.fillna(0 )<count_values>
import xgboost as xgb
print(data.Embarked.value_counts())
 data.Embarked = data.Embarked.fillna('S' )<feature_engineering>
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
data.Cabin = data.Cabin.fillna('Unknown_Cabin')
 data['Cabin'] = data['Cabin'].str[0]<count_values>
standard_scaler = StandardScaler()
full['Age']= pd.DataFrame(standard_scaler.fit_transform(full['Age'].values.reshape(-1,1)) )
print(data.Cabin.value_counts() )<count_values>
min_max_scaler = MinMaxScaler()
full['Fare']= pd.DataFrame(min_max_scaler.fit_transform(full['Fare'].values.reshape(-1,1)))
 full.head()
data.groupby('Pclass' ).Cabin.value_counts()<feature_engineering>

 familyDf['Family_Single'] = familyDf['FamilySize'].map(lambda s:1 if s==1 else 0)
 familyDf['Family_Small'] = familyDf['FamilySize'].map(lambda s:1 if s>=2 and s<=4 else 0)
 familyDf['Family_Large'] = familyDf['FamilySize'].map(lambda s:1 if s>=5 else 0)
 familyDf.head()
data['Cabin'] = np.where(( data.Pclass==1)&(data.Cabin=='U'),'C',
np.where(( data.Pclass==2)&(data.Cabin=='U'),'D',
np.where(( data.Pclass==3)&(data.Cabin=='U'),'G',
np.where(data.Cabin=='T','C',data.Cabin))))<drop_column>
full = pd.concat([full,familyDf],axis=1)
 full.drop(['SibSp','Parch'],axis=1,inplace=True)
 full.head()
data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(data['Title'], data['Sex'])
 data = data.drop('Name',axis=1 )<feature_engineering>
embarkedDf = pd.DataFrame()
embarkedDf = pd.get_dummies(full['Embarked'],prefix='Embarked')
 embarkedDf.head()
data['Title'] = np.where(( data.Title=='Capt')|(data.Title=='Countess')|(data.Title=='Don')|(data.Title=='Dona')
 |(data.Title=='Jonkheer')|(data.Title=='Lady')|(data.Title=='Sir')|(data.Title=='Major')|(data.Title=='Rev')|(data.Title=='Col'),'Other',data.Title)
 data['Title'] = data['Title'].replace('Ms','Miss')
 data['Title'] = data['Title'].replace('Mlle','Miss')
 data['Title'] = data['Title'].replace('Mme','Mrs' )<groupby>
full = pd.concat([full,embarkedDf],axis=1)
 full.drop(['Embarked'],axis=1,inplace=True)
 full.head()
data.groupby('Title' ).Age.mean()<feature_engineering>
pclassDf = pd.DataFrame()
pclassDf = pd.get_dummies(full['Pclass'],prefix='Pclass')
 pclassDf.head()
data['Age'] = np.where(( data.Age.isnull())&(data.Title=='Master'),5,
np.where(( data.Age.isnull())&(data.Title=='Miss'),22,
np.where(( data.Age.isnull())&(data.Title=='Mr'),32,
np.where(( data.Age.isnull())&(data.Title=='Mrs'),37,
np.where(( data.Age.isnull())&(data.Title=='Other'),45,
np.where(( data.Age.isnull())&(data.Title=='Dr'),44,data.Age))))))<feature_engineering>
full = pd.concat([full,pclassDf],axis=1)
 full.drop(['Pclass'],axis=1,inplace=True)
 full.head()
data['FamilySize'] = data.SibSp + data.Parch + 1
data['Mother'] = np.where(( data.Title=='Mrs')&(data.Parch >0),1,0)
 data['Free'] = np.where(data['Fare']==0, 1,0)
 data = data.drop(['SibSp','Parch','Sex'],axis=1 )<count_values>
titleDf = pd.DataFrame()
titleDf['Title'] = full['Name'].map(getTitle)
 titleDf.head()
TypeOfTicket = []
for i in range(len(data.Ticket)) :
ticket = data.Ticket.iloc[i]
for c in string.punctuation:
ticket = ticket.replace(c,"")
 splited_ticket = ticket.split(" ")
 if len(splited_ticket)== 1:
TypeOfTicket.append('NO')
 else:
TypeOfTicket.append(splited_ticket[0])
 data['TypeOfTicket'] = TypeOfTicket
data.TypeOfTicket.value_counts()
data['TypeOfTicket'] = np.where(( data.TypeOfTicket!='NO')&(data.TypeOfTicket!='PC')&(data.TypeOfTicket!='CA')& 
(data.TypeOfTicket!='A5')&(data.TypeOfTicket!='SOTONOQ'),'other',data.TypeOfTicket)
 data = data.drop('Ticket',axis=1 )<count_values>
titleDf['Title'].value_counts()
counting_values(data, 'Title','Survived' )<import_modules>
title_mapDict = {'Capt':'Officer',
'Col':'Officer',
'Major':'Officer',
'Jonkheer':'Royalty',
'Don':'Royalty',
'Sir':'Royalty',
'the Countess':'Royalty',
'Dona':'Royalty',
'Dr':'Officer',
'Rev':'Officer',
'Lady':'Royalty',
'Mr':'Mr',
'Miss':'Miss',
'Mrs':'Mrs',
'Master':'Master',
'Mlle':'Miss',
'Mme':'Mrs',
'Ms':'Mrs'
}
TypeOfTicket vs Survived<count_values>
titleDf['Title'] = titleDf['Title'].map(title_mapDict)
 titleDf = pd.get_dummies(titleDf['Title'])
 titleDf.head()
counting_values(data, 'TypeOfTicket','Survived' )<count_values>
full = pd.concat([full,titleDf],axis=1)
 full.drop(['Name'],axis=1,inplace=True)
 full.head()
counting_values(data, 'Cabin','Survived' )<categorify>
full = pd.concat([full,cabinDf],axis=1)
 full.drop(['Cabin'],axis=1,inplace=True)
 full.head()
data = pd.get_dummies(data)
 data.head(10 )<split>
full['Ticket'] = full['Ticket'].map(getTicket)
 ticketDf = pd.get_dummies(full['Ticket'],prefix='Ticket')
 full = pd.concat([full,ticketDf],axis=1)
 full.drop(['Ticket'],axis=1,inplace=True)
 full.head()
trainX, testX, trainY, testY = train_test_split(data[data.Survived.isnull() ==False].drop('Survived',axis=1),data.Survived[data.Survived.isnull() ==False],test_size=0.30, random_state=2019 )<create_dataframe>
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
Results = pd.DataFrame({'Model': [],'Accuracy Score': [], 'Recall':[], 'F1score':[]})
<compute_train_metric>
full_noid = full.drop(['PassengerId'],axis=1 )
model = DecisionTreeClassifier(max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['DecisionTreeClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )<compute_test_metric>
def reverse(full):
full_c = full.copy(deep=True)
 type(full_c)
 train_num= 891
train_X = full_c.drop(columns ='Survived',axis=1 ).loc[0:train_num-1,:]
train_y = full.loc[0:train_num-1,'Survived']
pred_X = full_c.drop(columns ='Survived',axis=1 ).loc[train_num:,:]
return train_X,train_y,pred_X
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )<prepare_output>
train_X,train_y,pred_X = reverse(full_noid )
model = RandomForestClassifier(n_estimators=2500, max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['RandomForestClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )<compute_test_metric>
clf = RandomForestClassifier(n_estimators=100,max_features='sqrt')
 clf = clf.fit(train_X,train_y )
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )<choose_model_class>
slf = SelectFromModel(clf,prefit=True)
 train_X_new = slf.transform(train_X)
 print('特征选择前训练特征个数：',train_X.shape[1],
'
特征选择后训练特征个数：',train_X_new.shape[1])
 pred_X_new = slf.transform(pred_X)
 print('特征选择前预测特征个数：',pred_X.shape[1],
'
特征选择后预测特征个数：',pred_X_new.shape[1] )
model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['XGBClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )<compute_test_metric>
from sklearn.linear_model import LogisticRegression,LogisticRegressionCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble.gradient_boosting import GradientBoostingClassifier
from sklearn.svm import SVC
from xgboost import XGBClassifier,XGBRFClassifier
from sklearn.model_selection import cross_val_score
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )<prepare_x_and_y>
lr = LogisticRegression()
rfc = RandomForestClassifier()
gboosting = GradientBoostingClassifier()
svc = SVC()
xgb = XGBClassifier()
xgbrfc = XGBRFClassifier()
models = [lr,rfc,gboosting,svc,xgb,xgbrfc]
trainX = data[data.Survived.isnull() ==False].drop(['Survived','train'],axis=1)
 trainY = data.Survived[data.Survived.isnull() ==False]
testX = data[data.Survived.isnull() ==True].drop(['Survived','train'],axis=1)
 model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 test = data[data.train==0]
test['Survived'] = model.predict(testX ).astype(int)
 test = test.reset_index()
test[['PassengerId','Survived']].to_csv("submissionXGB.csv",index=False)
 print("done1" )<set_options>
for model in models:
print('Cross validation of:{0}'.format(model.__class__))
score = cross_val_score(model,X_train,y_train,cv=5,scoring='accuracy')
 score = score.mean()
print('score = {0}'.format(score))
%matplotlib inline 
<load_from_csv>
from sklearn.model_selection import GridSearchCV
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 combine = [train_df , test_df]<sort_values>
from sklearn.model_selection import GridSearchCV
train_df.groupby('Sex' ).Survived.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<sort_values>
param_test1 = {
'n_estimators':range(20,81,10)
 }
gsearch1 = GridSearchCV(estimator=GradientBoostingClassifier(
 learning_rate=0.1,
max_depth=6,
min_samples_split=5,
min_samples_leaf=2,
max_features='sqrt',
subsample=0.8,
random_state=10 
),param_grid=param_test1,scoring="accuracy",n_jobs=4,iid=False,cv=5 
)
 gsearch1.fit(X_train,y_train)
 n_estimators = gsearch1.best_params_['n_estimators']
print(gsearch1.scoring)
 print(gsearch1.best_params_)
 print(gsearch1.best_score_ )
train_df.groupby('Ticket' ).Survived.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<sort_values>
param_test2 = {
'max_depth':range(2,8),
'min_samples_split':range(2,10)
 }
gsearch2 = GridSearchCV(estimator=GradientBoostingClassifier(
 learning_rate=0.1,
n_estimators=n_estimators,
min_samples_leaf=2,
max_features='sqrt',
subsample=0.8,
random_state=10 
),param_grid=param_test2,scoring="accuracy",n_jobs=4,iid=False,cv=5 
)
 gsearch2.fit(X_train,y_train)
 max_depth = gsearch2.best_params_['max_depth']
print(gsearch2.scoring)
 print(gsearch2.best_params_)
 print(gsearch2.best_score_ )
train_df.groupby('Embarked' ).Survived.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<count_values>
param_test3 = {
'min_samples_split':range(2,10),
'min_samples_leaf':range(2,20)
 }
gsearch3 = GridSearchCV(estimator=GradientBoostingClassifier(
 learning_rate=0.1,
max_depth=max_depth,
n_estimators=n_estimators,
max_features='sqrt',
subsample=0.8,
random_state=10 
),param_grid=param_test3,scoring="accuracy",n_jobs=4,iid=False,cv=5 
)
 gsearch3.fit(X_train,y_train)
 min_samples_split=gsearch3.best_params_['min_samples_split']
min_samples_leaf=gsearch3.best_params_['min_samples_leaf']
print(gsearch3.scoring)
 print(gsearch3.best_params_)
 print(gsearch3.best_score_ )
train_df.Cabin.value_counts()<define_variables>
param_test4 = {
'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]
}
gsearch4 = GridSearchCV(estimator=GradientBoostingClassifier(
 learning_rate=0.1,
max_depth=max_depth,
n_estimators=n_estimators,
max_features='sqrt',
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
random_state=10 
),param_grid=param_test4,scoring="accuracy",n_jobs=4,iid=False,cv=5 
)
 gsearch4.fit(X_train,y_train)
 subsample=gsearch4.best_params_['subsample']
print(gsearch4.scoring)
 print(gsearch4.best_params_)
 print(gsearch4.best_score_ )
features_df = train_df.drop('Survived', axis=1)
 num_features = features_df.select_dtypes(np.number)
 num_features.sample(5 )<sort_values>
gbm=GradientBoostingClassifier(
 learning_rate=0.01,
max_depth=max_depth,
n_estimators=n_estimators*10,
max_features='sqrt',
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
subsample=subsample,
random_state=10 
)
 gbm.fit(train_X_new,train_y )
train_df.groupby('Pclass' ).Survived.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<sort_values>
pred_y = gbm.predict(pred_X_new )
train_df.groupby('Sex' ).Pclass.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<sort_values>

 pred_y = pred_y.astype(int )
train_df.groupby('Fare' ).Survived.agg(['mean' , 'count'] ).sort_values('count' , ascending = False )<categorify>
passengerId = full.loc[891:,'PassengerId']
predDf = pd.DataFrame(
 {'PassengerId':passengerId,
'Survived':pred_y})
 predDf.shape
<split><EOS> 
predDf.to_csv('titanic_pred.csv',index=False )