Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
np.random.seed(1234 )
x = train.budget.values.reshape(-1,1)
 y = train.revenue
reg = LinearRegression().fit(x, y )<compute_test_metric>
dataset = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 y = dataset['Survived'].values
dataset.head()
print(f'Regression Score: {reg.score(x, y)}')
 print(f'Regression Coefficient: {reg.coef_[0]}')
 print(f'Regression Intercept: {reg.intercept_}' )<predict_on_test>
dataset['Cabin'].fillna('NotKnown',inplace=True)
 dataset['Age'].fillna(dataset['Age'].mean() ,inplace=True)
 dataset['Embarked'].fillna('S',inplace=True)
 test['Cabin'] = test['Cabin'].fillna('NotKnown')
 test['Age'] = test['Age'].fillna(test['Age'].mean())
predictions = reg.predict(test['budget'].values.reshape(-1,1))<feature_engineering>
print(np.unique([s.split(',')[1].split(' ')[1] for s in dataset['Name']]))
print(np.unique([s.split(',')[1].split(' ')[1] for s in test['Name']]))
dataset['Title'] = [s.split(',')[1].split(' ')[1] for s in dataset['Name']]
test['Title'] = [s.split(',')[1].split(' ')[1] for s in test['Name']]
dataset.head()
submission['revenue'] = np.round(np.expm1(predictions))<save_to_csv>
dataset['Title'].replace(['Capt.'],'Board',inplace=True)
 dataset['Title'].replace(['Don.','Dona.','Lady.','Jonkheer.','Rev.','Sir.','the'],'Hon',inplace=True)
 dataset['Title'].replace(['Col.','Dr.','Major.','Master.'],'Professionals',inplace=True)
 dataset['Title'].replace(['Miss.','Mlle.','Mme.','Mr.','Mrs.','Ms.'],'NoHon', inplace=True)
 test['Title'].replace(['Capt.'],'Board',inplace=True)
 test['Title'].replace(['Don.','Dona.','Lady.','Jonkheer.','Rev.','Sir.','the'],'Hon',inplace=True)
 test['Title'].replace(['Col.','Dr.','Major.','Master.'],'Professionals',inplace=True)
 test['Title'].replace(['Miss.','Mlle.','Mme.','Mr.','Mrs.','Ms.'],'NoHon', inplace=True )
submission.to_csv('submission_budget_linreg.csv', index = False )<create_dataframe>
print(np.unique([x.split(' ')[0] if not(x.isdigit())else 'OnlyNumber' for x in dataset['Ticket']]))
dataset['Ticket'] = [x.split(' ')[0] if not(x.isdigit())else 'OnlyNumber' for x in dataset['Ticket']]
test['Ticket'] = [x.split(' ')[0] if not(x.isdigit())else 'OnlyNumber' for x in test['Ticket']]
rf_cols = ['budget', 'original_language', 'popularity', 'release_date', 'runtime', 'status', 'homepage', 'overview', 'revenue']
rf_train = train[rf_cols].copy()
rf_cols.remove('revenue')
 rf_test = test[rf_cols].copy()<feature_engineering>
print('before(train)-> '+ str(len(dataset.loc[dataset.Fare == 0.0]['Fare'])))
 print('before(test)-> '+ str(len(test.loc[test.Fare == 0.0]['Fare'])))
 dataset['Fare'].replace(0.0,np.nan,inplace=True)
 test['Fare'].replace(0.0,np.nan,inplace=True)
 meanFirst = dataset.loc[dataset.Pclass==1]['Fare'].mean()
print('First Class mean is ' + str(meanFirst))
meanSecond = dataset.loc[dataset.Pclass==2]['Fare'].mean()
print('Second Class mean is ' + str(meanSecond))
meanThird = dataset.loc[dataset.Pclass==3]['Fare'].mean()
print('Third Class mean is ' + str(meanThird))
means=[meanFirst,meanSecond,meanThird]
fcd = dataset.loc[dataset.Pclass == 1].index
scd = dataset.loc[dataset.Pclass == 2].index
tcd = dataset.loc[dataset.Pclass == 3].index
fct = test.loc[test.Pclass == 1].index
sct = test.loc[test.Pclass == 2].index
tct = test.loc[test.Pclass == 3].index
dataset.loc[fcd,'Fare'] = dataset.loc[fcd,'Fare'].fillna(means[0])
 dataset.loc[scd,'Fare'] = dataset.loc[scd,'Fare'].fillna(means[1])
 dataset.loc[tcd,'Fare'] = dataset.loc[tcd,'Fare'].fillna(means[2])
 test.loc[fct,'Fare'] = test.loc[fct,'Fare'].fillna(means[0])
 test.loc[sct,'Fare'] = test.loc[sct,'Fare'].fillna(means[1])
 test.loc[tct,'Fare'] = test.loc[tct,'Fare'].fillna(means[2])
 print('after(train)-> '+ str(len(dataset.loc[dataset.Fare == 0]['Fare'])))
 print('after(test)-> '+ str(len(test.loc[dataset.Fare == 0]['Fare'])))
 print('after(train)null -> '+ str(dataset['Fare'].count() -891))
print('after(test)null -> '+ str(test['Fare'].count() - 418))
minFare = 0
qntl25 = dataset['Fare'].quantile(0.25)
 qntl50 = dataset['Fare'].quantile(0.5)
 qntl75 = dataset['Fare'].quantile(0.75)
 qntplus = dataset['Fare'].quantile(0.75)+ 1.5*(qntl75-qntl50)
 maxFare = 1e6
dataset['Fare'] = pd.cut(dataset['Fare'], bins= [minFare,qntl25, qntl75,qntplus,maxFare], labels=['Low','Medium','High','VeryHigh'])
 test['Fare'] = pd.cut(test['Fare'], bins= [minFare,qntl25, qntl75,qntplus,maxFare], labels=['Low','Medium','High','VeryHigh'])
 sns.barplot(data=dataset, x='Fare', y='Survived' )
def fill_budget(df, median_budget):
df['budget_is_median'] = 0
df.loc[df.budget == 0, 'budget_is_median'] = 1
df.loc[df.budget == 0, 'budget'] = median_budget
return df<drop_column>
encSex = preprocessing.OrdinalEncoder().fit(pd.DataFrame(dataset['Sex']))
encTitle = preprocessing.OrdinalEncoder().fit(pd.DataFrame(dataset['Title']))
encParchSibSp = preprocessing.OrdinalEncoder().fit(pd.DataFrame(dataset['Parch-SibSp']))
encFare = preprocessing.OrdinalEncoder().fit(pd.DataFrame(dataset['Fare']))
encEmbark = preprocessing.OneHotEncoder().fit(pd.DataFrame(dataset['Embarked']))
dataset['Sex']= encSex.transform(pd.DataFrame(dataset['Sex']))
dataset['Title']= encTitle.transform(pd.DataFrame(dataset['Title']))
dataset['Parch-SibSp']= encParchSibSp.transform(pd.DataFrame(dataset['Parch-SibSp']))
dataset['Fare']= encFare.transform(pd.DataFrame(dataset['Fare']))
dataset = pd.concat([dataset,pd.DataFrame(encEmbark.transform(pd.DataFrame(dataset['Embarked'])).toarray() ,columns=['Embark_C','Embark_Q','Embark_S'])], axis=1)
 dataset.drop(columns=['Embarked'],inplace=True)
 test = pd.concat([test,pd.DataFrame(encEmbark.transform(pd.DataFrame(test['Embarked'])).toarray() ,columns=['Embark_C','Embark_Q','Embark_S'])], axis=1)
 test.drop(columns=['Embarked'],inplace=True)
 test['Sex'] = encSex.transform(pd.DataFrame(test['Sex']))
test['Title'] = encTitle.transform(pd.DataFrame(test['Title']))
test['Parch-SibSp'] = encParchSibSp.transform(pd.DataFrame(test['Parch-SibSp']))
test['Fare']= encFare.transform(pd.DataFrame(test['Fare']))
rf_train = fill_budget(rf_train, median_budget)
 rf_test = fill_budget(rf_test, median_budget )<concatenate>
dataset.drop(columns=['Cabin','PassengerId','Name','Ticket','Survived'], inplace=True)
 test.drop(columns=['Cabin','PassengerId','Name','Ticket'], inplace=True )
rf_combined = pd.concat([rf_train, rf_test], sort=False )<categorify>
if sel_threshold >= 0.05 and sel_threshold < 0.1:
dataset.drop(columns=['Embark_S','Embark_Q','Embark_C','Title'],inplace=True)
 test.drop(columns=['Embark_S','Embark_Q','Embark_C','Title'],inplace=True)
 if sel_threshold >= 0.1 and sel_threshold < 0.15:
dataset.drop(columns=['Embark_S','Embark_Q','Embark_C','Parch-SibSp','Fare','Title'],inplace=True)
 test.drop(columns=['Embark_S','Embark_Q','Embark_C','Parch-SibSp','Fare','Title'],inplace=True)
 if sel_threshold == 0.15:
dataset.drop(columns=['Embark_S','Embark_Q','Embark_C','Parch-SibSp','Pclass','Title','Fare'],inplace=True)
 test.drop(columns=['Embark_S','Embark_Q','Embark_C','Parch-SibSp','Pclass','Title','Fare'],inplace=True )
le = LabelEncoder()
le.fit(rf_combined['original_language'])
 rf_train['original_language'] = le.transform(rf_train['original_language'])
 rf_test['original_language'] = le.transform(rf_test['original_language'] )<feature_engineering>
scaler = preprocessing.StandardScaler()
X = scaler.fit_transform(dataset)
 test_X = scaler.transform(test )
rf_test.loc[rf_test['status'].isnull() == True, 'status'] = 'Released'
rf_combined.loc[rf_combined['status'].isnull() == True, 'status'] = 'Released'<categorify>
cv_size = 5
le = LabelEncoder()
le.fit(rf_combined['status'])
 rf_train['status'] = le.transform(rf_train['status'])
 rf_test['status'] = le.transform(rf_test['status'] )<feature_engineering>
lr_model = linear_model.LogisticRegression()
scores_lr = model_selection.cross_validate(lr_model,X,y,cv=cv_size, scoring=('accuracy','f1'))
print('Trained in ' + '{:.2f}'.format(scores_lr['fit_time'].mean())+ 's , achieved Accuracy of ' + '{:.3f}'.format(scores_lr['test_accuracy'].mean())+ '('+ '{:.3f}'.format(scores_lr['test_accuracy'].std())+')' )
rf_train.loc[rf_train['homepage'].isnull() == True, 'homepage'] = 0
rf_train.loc[rf_train['homepage'].isnull() == False, 'homepage'] = 1
rf_test.loc[rf_test['homepage'].isnull() == True, 'homepage'] = 0
rf_test.loc[rf_test['homepage'].isnull() == False, 'homepage'] = 1<feature_engineering>
lr_model = linear_model.LogisticRegression()
def fill_runtime(df, median_runtime):
df['runtime_is_median'] = 0
df.loc[df.runtime == 0, 'runtime_is_median'] = 1
df.loc[df.runtime.isnull() == True, 'runtime_is_median'] = 1
df.loc[df.runtime == 0, 'runtime'] = median_runtime
df.loc[df.runtime.isnull() == True, 'runtime'] = median_runtime
return df<drop_column>
results_rf = []
indexes = []
for j in range(100,300,100):
for i in range(2,10):
for k in range(2,10):
indexes.append('n'+str(j)+'d'+str(i)+'s'+str(k)) ;
rf_model = ensemble.RandomForestClassifier(n_estimators=j,max_depth=i,min_samples_split=k, n_jobs=-1,oob_score = True)
 scores_rf = rf_model.fit(X,y ).oob_score_
results_rf.append({'acc_mean': scores_rf})
 results_rf = pd.DataFrame(results_rf, index=indexes )
rf_train = fill_runtime(rf_train, median_runtime)
 rf_test = fill_runtime(rf_test, median_runtime )<import_modules>
rf_model = ensemble.RandomForestClassifier(n_estimators=100,max_depth=4,min_samples_split=4, n_jobs=-1 )
from datetime import timedelta, date<feature_engineering>
results_et = []
indexes = []
for j in range(100,300,100):
for i in range(2,10):
for k in range(2,10):
indexes.append('n'+str(j)+'d'+str(i)+'s'+str(k)) ;
et_model = ensemble.ExtraTreesClassifier(n_estimators=j,max_depth=i,min_samples_split=k,n_jobs=-1,oob_score=True,bootstrap=True)
 scores_et = et_model.fit(X,y ).oob_score_
results_et.append({'acc_mean': scores_et})
 results_et = pd.DataFrame(results_et, index=indexes )
rf_test.loc[rf_test['release_date'].isnull() == True, 'release_date'] = '10/19/2001'
test.loc[test['release_date'].isnull() == True, 'release_date'] = '10/19/2001'<feature_engineering>
et_model = ensemble.ExtraTreesClassifier(n_estimators=200,max_depth=6,min_samples_split=4,n_jobs=-1,bootstrap=True )
def add_date_features(df, col, prefix):
df[col] = pd.to_datetime(df[col])
 future = df[col] > pd.Timestamp(year=2017,month=12,day=31)
 df.loc[future, col] -= timedelta(days=365.25*100)
 df[prefix+'_day_of_week'] = df[col].dt.dayofweek
df[prefix+'_day_of_year'] = df[col].dt.dayofyear
df[prefix+'_month'] = df[col].dt.month
df[prefix+'_year'] = df[col].dt.year
df[prefix+'_day'] = df[col].dt.day
df[prefix+'_is_year_end'] = df[col].dt.is_year_end
df[prefix+'_is_year_start'] = df[col].dt.is_year_start
df[prefix+'_week'] = df[col].dt.week
df[prefix+'_quarter'] = df[col].dt.quarter
df.drop(col, axis = 1, inplace = True)
 return df<feature_engineering>
results_svc = []
for i in np.arange(0.1,10,0.1):
svm_model = svm.SVC(C=i)
 scores_svc = model_selection.cross_validate(svm_model,X,y,cv=cv_size, scoring=('accuracy','f1'))
results_svc.append({'acc_mean': scores_svc['test_accuracy'].mean() , 'acc_std': scores_svc['test_accuracy'].std() , 'f1_mean': scores_svc['test_f1'].mean() })
 results_svc = pd.DataFrame(results_svc, index=np.arange(0.1,10,0.1))
rf_train = add_date_features(rf_train, 'release_date', 'release')
 rf_test = add_date_features(rf_test, 'release_date', 'release' )<categorify>
svm_model = svm.SVC(C=0.5 )
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d<feature_engineering>
results_knn = []
for k in range(5,50):
knn_model = neighbors.KNeighborsClassifier(n_neighbors=k, weights='uniform',algorithm='ball_tree')
 scores_knn = model_selection.cross_validate(knn_model,X,y,cv=cv_size,scoring=('accuracy','f1'))
results_knn.append({'acc_mean': scores_knn['test_accuracy'].mean() , 'acc_std': scores_knn['test_accuracy'].std() , 'f1_mean': scores_knn['test_f1'].mean() })
 results_knn = pd.DataFrame(results_knn, index=range(5,50))
json_cols = ['production_companies', 'production_countries', 'cast', 'crew', 'Keywords', 'belongs_to_collection']
for col in json_cols:
rf_train[col] = train[col]
rf_train[col] = rf_train[col].apply(lambda x: get_dictionary(x))
rf_test[col] = test[col]
rf_test[col] = rf_test[col].apply(lambda x: get_dictionary(x))<categorify>
knn_model = neighbors.KNeighborsClassifier(n_neighbors=10, weights='uniform',algorithm='ball_tree' )
for col in json_cols:
if col == 'belongs_to_collection':
rf_train['collection_name'] = rf_train[col].apply(lambda row: row[0]['name'] if row != {} else '0')
 rf_test['collection_name'] = rf_test[col].apply(lambda row: row[0]['name'] if row != {} else '0')
 rf_combined = pd.concat([rf_train, rf_test], sort=False)
 le = LabelEncoder()
le.fit(rf_combined['collection_name'])
 rf_train['collection_name'] = le.transform(rf_train['collection_name'])
 rf_test['collection_name'] = le.transform(rf_test['collection_name'])
 rf_train[col] = rf_train[col].apply(lambda row: 0 if row is None else len(row))
rf_test[col] = rf_test[col].apply(lambda row: 0 if row is None else len(row))
rf_train['overview_wordcount'] = rf_train['overview'].str.split().str.len()
rf_train.drop('overview', axis = 1, inplace = True)
 rf_train.loc[rf_train['overview_wordcount'].isnull() == True, 'overview_wordcount'] = 0
rf_test['overview_wordcount'] = rf_test['overview'].str.split().str.len()
rf_test.drop('overview', axis = 1, inplace = True)
 rf_test.loc[rf_test['overview_wordcount'].isnull() == True, 'overview_wordcount'] = 0
rf_train['_budget_runtime_ratio'] = np.round(rf_train['budget']/rf_train['runtime'], 2)
 rf_train['_budget_year_ratio'] = np.round(rf_train['budget']/(rf_train['release_year']*rf_train['release_year']), 2)
 rf_train['_releaseYear_popularity_ratio'] = np.round(rf_train['release_year']/rf_train['popularity'], 2)
 rf_test['_budget_runtime_ratio'] = np.round(rf_test['budget']/rf_test['runtime'], 2)
 rf_test['_budget_year_ratio'] = np.round(rf_test['budget']/(rf_test['release_year']*rf_test['release_year']), 2)
 rf_test['_releaseYear_popularity_ratio'] = np.round(rf_test['release_year']/rf_test['popularity'], 2)
<split>
nb_model = naive_bayes.GaussianNB()
scores_nb = model_selection.cross_validate(nb_model,X,y,cv=cv_size, scoring=('accuracy','f1'))
print('Trained in ' + '{:.2f}'.format(scores_nb['fit_time'].mean())+ 's , achieved Accuracy of ' + '{:.3f}'.format(scores_nb['test_accuracy'].mean())+ '('+ '{:.3f}'.format(scores_nb['test_accuracy'].std())+')')
X_train, X_valid, y_train, y_valid = train_test_split(
 rf_train.drop('revenue', axis = 1), rf_train['revenue'],
test_size=0.1,
random_state=42
)<compute_test_metric>
nb_model = naive_bayes.GaussianNB()
def rmse(y_pred, y_true):
return np.sqrt(mean_squared_error(y_pred, y_true))
def print_rf_score(model):
print(f'Train R2: {model.score(X_train, y_train)}')
 print(f'Valid R2: {model.score(X_valid, y_valid)}')
 print(f'Train RMSE: {rmse(model.predict(X_train), y_train)}')
 print(f'Valid RMSE: {rmse(model.predict(X_valid), y_valid)}' )<train_model>
stacked_model = ensemble.StackingClassifier(estimators=[('et',et_model), 
('rf',rf_model), 
('svm',svm_model), 
('lr',lr_model)
 ])
 voting_model = ensemble.VotingClassifier(estimators=[('et',et_model), 
('rf',rf_model), 
('svm',svm_model), 
('lr',lr_model)
 ])
 scores_stack = model_selection.cross_validate(stacked_model,X,y,cv=cv_size,scoring='accuracy')
 scores_voting = model_selection.cross_validate(voting_model,X,y,cv=cv_size,scoring='accuracy')
 print('Trained in ' + '{:.2f}'.format(scores_stack['fit_time'].mean())+ 's , achieved Accuracy of ' + '{:.3f}'.format(scores_stack['test_score'].mean())+ '('+ '{:.3f}'.format(scores_stack['test_score'].std())+')')
 print('Trained in ' + '{:.2f}'.format(scores_voting['fit_time'].mean())+ 's , achieved Accuracy of ' + '{:.3f}'.format(scores_voting['test_score'].mean())+ '('+ '{:.3f}'.format(scores_voting['test_score'].std())+')' )
rf = RandomForestRegressor(n_jobs = -1, random_state = 42)
 rf.fit(X_train, y_train)
 print_rf_score(rf )<train_model>
models = {'lr':lr_model,
'rf':rf_model,
'et':et_model,
'svm': svm_model,
'knn': knn_model,
'nb': nb_model,
'stack': stacked_model,
'vote':voting_model}
<set_options><EOS> 
for i in models.keys() :
m = models[i].fit(X,y)
 y_pred = m.predict(test_X)
 prediction = pd.read_csv('.. /input/titanic/gender_submission.csv')
 prediction['Survived'] = y_pred
prediction.set_index('PassengerId')
 prediction.to_csv('submission-'+i+'.csv',index = False )