Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42
test = pd.read_csv('.. /input/test.csv' )<create_dataframe>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['Survived'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )
submission = pd.DataFrame(
 {'key': test.key, 'fare_amount': pres.reshape(9914)},
columns = ['key', 'fare_amount'] )<save_to_csv>
def display_missing(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in dfs:
print('{}'.format(df.name))
display_missing(df )
submission.to_csv('submission.csv', index = False )<set_options>
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))
print('Median age of all passengers: {}'.format(df_all['Age'].median()))
df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
%matplotlib inline
sns.set_style("white")
 warnings.filterwarnings('ignore')
<load_from_csv>
df_all[df_all['Embarked'].isnull() ]
data = pd.read_csv(".. /input/new-york-city-taxi-fare-prediction/train.csv", nrows = 7000000 )<define_variables>
df_all['Embarked'] = df_all['Embarked'].fillna('S' )
q1 = data['fare_amount'].quantile(0.25)
 q3 = data['fare_amount'].quantile(0.75)
 iqr = q3 - q1
print("Fare Amount lower bound : ", q1 -(1.5 * iqr),
"Fare Amount upper bound : ", q3 +(1.5 * iqr))<count_missing_values>
df_all[df_all['Fare'].isnull() ]
print("Total null values:
", data.isnull().sum())
 print("Percentage of null values:
",
data[["dropoff_longitude", "dropoff_latitude"]].isnull().sum() / data.shape[0] )<drop_column>
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )
data.dropna(how='any', axis='rows', inplace=True )<categorify>
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'
def get_cleaned(df):
return df[(df.fare_amount > 0)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -68)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.pickup_longitude > -80)&(df.dropoff_longitude < -68)& 
(df.passenger_count > 0)&(df.passenger_count < 8)]
data = get_cleaned(data)
 print(len(data))
print("Data lost after the cleaning process: ", 7000000 - len(data))<compute_test_metric>
df_all_decks_survived = df_all.groupby(['Deck', 'Survived'] ).count().drop(columns=['Sex', 'Age', 'SibSp', 'Parch', 'Fare',
'Embarked', 'Pclass', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
def get_survived_dist(df):
surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}
decks = df.columns.levels[0]
for deck in decks:
for survive in range(0, 2):
surv_counts[deck][survive] = df[deck][survive][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
return surv_counts, surv_percentages
def display_surv_dist(percentages):
df_survived_percentages = pd.DataFrame(percentages ).transpose()
deck_names =('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')
 bar_count = np.arange(len(deck_names))
bar_width = 0.85
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(20, 10))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Deck', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage in Decks', size=18, y=1.05)
 plt.show()
all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)
 display_surv_dist(all_surv_per )
def sphere_dist(pick_lat, pick_lon, drop_lat, drop_lon):
R_earth = 6371
pick_lat, pick_lon, drop_lat, drop_lon = map(np.radians, [pick_lat, pick_lon,
drop_lat, drop_lon])
 dlat = drop_lat - pick_lat
dlon = drop_lon - pick_lon
a = np.sin(dlat/2.0)**2 + np.cos(pick_lat)* np.cos(drop_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))<feature_engineering>
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()
def datetime_info(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC")
 df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.day
df['month'] = df.pickup_datetime.dt.month
df['weekday'] = df.pickup_datetime.dt.weekday
df['year'] = df.pickup_datetime.dt.year
return df
data = datetime_info(data)
 data = airport_dist(data)
 data['distance'] = sphere_dist(data['pickup_latitude'], data['pickup_longitude'],
data['dropoff_latitude'], data['dropoff_longitude'])
 data.head()<drop_column>
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
display_missing(df )
data.drop(columns=["key", "pickup_datetime"], inplace=True)
 data.head()<split>
corr = df_train_corr_nd['Correlation Coefficient'] > 0.1
df_train_corr_nd[corr]
y = data["fare_amount"]
train = data.drop(columns=["fare_amount"])
 x_train, x_test, y_train, y_test = train_test_split(train, y, random_state=2666, test_size=0.05 )<init_hyperparams>
corr = df_test_corr_nd['Correlation Coefficient'] > 0.1
df_test_corr_nd[corr]
params = {
"max_depth": 7,
"subsample": 0.9,
"eta": 0.03,
"colsample_bytree": 0.9,
"random_state": 2666,
"objective": "reg:linear",
"eval_metric": "rmse",
"silent": 1
}<train_model>
df_all = concat_df(df_train, df_test)
 df_all.head()
def XGBmodel(x_train, x_test, y_train, y_test, params):
matrix_train = xgb.DMatrix(x_train, label=y_train)
 matrix_test = xgb.DMatrix(x_test, label=y_test)
 model = xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
start_time = time.time()
model = XGBmodel(x_train, x_test, y_train, y_test, params )<define_variables>
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )
time_taken = time.time() - start_time
time_taken<load_from_csv>
df_all['Age'] = pd.qcut(df_all['Age'], 10 )
test = pd.read_csv('.. /input/new-york-city-taxi-fare-prediction/test.csv')
 test = datetime_info(test)
 test = airport_dist(test)
 test['distance'] = sphere_dist(test['pickup_latitude'], test['pickup_longitude'],
test['dropoff_latitude'] , test['dropoff_longitude'])
 test_key = test['key']
x_pred = test.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit=model.best_ntree_limit )<save_to_csv>
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()<set_options>
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1
warnings.filterwarnings('ignore')
 pd.set_option('display.float_format', lambda x: '%.4f' % x)
 TRAIN_PATH = '.. /input/new-york-city-taxi-fare-prediction/train.csv'
TEST_PATH = '.. /input/new-york-city-taxi-fare-prediction/test.csv'<load_from_csv>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]
test = pd.read_csv(TEST_PATH)
 print('Null values:',test.isnull().sum().sum())
 test.head()<load_from_csv>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA
df_temp = pd.read_csv(TRAIN_PATH, nrows=100000)
 profile = pandas_profiling.ProfileReport(df_temp, title="Profile Report", minimal=True, progress_bar=False)
 profile.to_notebook_iframe()<data_type_conversions>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2
def clean_df(df):
df['pickup_datetime'] = df['pickup_datetime'].str.slice(0, 15)
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], utc=True, format='%Y-%m-%d %H:%M')
 df = df.assign(rev=df.dropoff_latitude<df.dropoff_longitude)
 idx =(df['rev'] == 1)
 df.loc[idx,['dropoff_longitude','dropoff_latitude']] = df.loc[idx,['dropoff_latitude','dropoff_longitude']].values
df.loc[idx,['pickup_longitude','pickup_latitude']] = df.loc[idx,['pickup_latitude','pickup_longitude']].values
criteria =(
 " 0 < fare_amount <= 500"
" and 0 < passenger_count <= 6 "
" and -75 <= pickup_longitude <= -72 "
" and -75 <= dropoff_longitude <= -72 "
" and 40 <= pickup_latitude <= 42 "
" and 40 <= dropoff_latitude <= 42 " 
)
 df =(df 
.dropna() 
.query(criteria)
.reset_index() 
.drop(columns=['rev', 'index'])
)
 return df
def load_df(nrows=None, features=None):
cols = [
'fare_amount', 'pickup_datetime','pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'passenger_count'
]
df_as_list = []
for df_chunk in pd.read_csv(TRAIN_PATH, usecols=cols, nrows=nrows, chunksize=5000000):
df_chunk = clean_df(df_chunk)
 if features == 'explore':
df_chunk = exploration_features(df_chunk)
 elif features == 'model':
df_chunk = modelling_features(df_chunk)
 else:
df_chunk = df_chunk.drop(columns='pickup_datetime')
 df_as_list.append(df_chunk)
 df = pd.concat(df_as_list)
 return df<split>
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )
def get_split_sets(train):
x = train.drop(columns=['fare_amount'])
 y = train['fare_amount'].values
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.1, random_state=0)
 return x_train, x_val, y_train, y_val
def lin_model(x_train, x_val, y_train, y_val):
model = LinearRegression()
model.fit(x_train, y_train)
 pred = model.predict(x_val)
 rmse = np.sqrt(mean_squared_error(y_val, pred))
return model, rmse, pred
def knn_model(x_train, x_val, y_train, y_val, neighbors):
min_rmse = 1000
for n in neighbors:
knn = KNeighborsRegressor(n_neighbors=n)
 knn.fit(x_train, y_train)
 pred = knn.predict(x_val)
 rmse = np.sqrt(mean_squared_error(y_val, pred))
if rmse < min_rmse:
min_rmse = rmse
model = knn
best_pred = pred
print('Neighbours', n, 'RMSE', rmse)
 return model, min_rmse, best_pred
def lgbm_model(params,x_train, x_val, y_train, y_val):
lgbm_train = lgbm.Dataset(x_train, y_train, silent=True)
 lgbm_val = lgbm.Dataset(x_val, y_val, silent=True)
 model = lgbm.train(params=params, train_set=lgbm_train, valid_sets=lgbm_val, verbose_eval=100)
 pred = model.predict(x_val, num_iteration=model.best_iteration)
 rmse = np.sqrt(mean_squared_error(y_val, pred))
return model, rmse, pred<load_from_csv>
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )
train = load_df(1000000)
 x_train, x_val, y_train, y_val= get_split_sets(train)
 test = pd.read_csv(TEST_PATH)
 x_test = test.drop(columns=['key'] )<compute_test_metric>
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()
lin_init_model, lin_init_rmse, lin_init_pred = lin_model(x_train, x_val, y_train, y_val )<find_best_params>
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
k_choices = [10,20,30,40,50,60]
knn_cols = ['pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']
knn_init, knn_init_rmse, knn_init_pred = knn_model(x_train[knn_cols], x_val[knn_cols], y_train, y_val, k_choices )<train_model>
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )
lgbm_params = {
'objective': 'regression',
'boosting': 'gbdt',
'num_leaves': 400,
'learning_rate': 0.1,
'max_bin': 3000,
'num_rounds': 5000,
'early_stopping_rounds': 100,
'metric' : 'rmse'
}
lgbm_init_model, lgbm_init_rmse, lgbm_init_pred = lgbm_model(lgbm_params, x_train, x_val, y_train, y_val )<train_model>
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=df_all.columns)
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
leaderboard_model.fit(X_train[trn_idx], y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(y_train[trn_idx], leaderboard_model.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(y_train[val_idx], leaderboard_model.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = leaderboard_model.feature_importances_
oob += leaderboard_model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, leaderboard_model.oob_score_))
print('Average OOB Score: {}'.format(oob))
print('Linear Regression RMSE', lin_init_rmse)
 print('KNN RMSE', knn_init_rmse)
 print('LightGBM RMSE', lgbm_init_rmse )<compute_test_metric>
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.5].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values
submission_df.to_csv('submissions.csv', header=True, index=False)
 submission_df.head(10 )