Kaggle competition: Google Analytics Customer Revenue Prediction
train.groupby('Pclass' ).mean() ['Survived']*100<groupby>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline
train.groupby(['Pclass', 'Sex'] ).Survived.mean()<count_values>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape
test['Titles'] = test['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 train['Titles'] = train['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 test['Titles'].value_counts()<count_values>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
train['Titles'].value_counts() 
<categorify>
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
train['Titles'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 test['Titles'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 train['Titles'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
 test['Titles'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
 train.groupby('Titles' ).Survived.mean() 
<feature_engineering>
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )
train['Ticket_letters'] = train.Ticket.apply(lambda x: x[:2])
 test['Ticket_letters'] = test.Ticket.apply(lambda x: x[:2])
 train['Ticket_length'] = train.Ticket.apply(lambda x: len(x))
test['Ticket_length'] = test.Ticket.apply(lambda x: len(x))<prepare_x_and_y>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
y = train['Survived']
features = ['Pclass', 'Fare', 'Titles', 'Embarked', 'Fam_group', 'Ticket_length', 'Ticket_letters']
X = train[features]
X.head()<categorify>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
numerical_cols = ['Fare']
categorical_cols = ['Pclass', 'Titles', 'Embarked', 'Fam_group', 'Ticket_length', 'Ticket_letters']
numerical_transformer = SimpleImputer(strategy='median')
 categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')) ,('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[('num', numerical_transformer, numerical_cols),('cat', categorical_transformer, categorical_cols)])
 titanic_pipeline = Pipeline(steps=[('preprocessor', preprocessor),('model', RandomForestClassifier(random_state=0, n_estimators=500, max_depth=5)) ])
 titanic_pipeline.fit(X,y)
 print('Cross validation score: {:.3f}'.format(cross_val_score(titanic_pipeline, X, y, cv=10 ).mean()))<predict_on_test>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
predictions = titanic_pipeline.predict(X_test )<save_to_csv>
', params)
 Best num_boost_round:', len(cv_results['rmse-mean']))
o = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': predictions})
 o.to_csv('my_submission.csv', index=False)
 print('Your submission was successfully saved!' )<install_modules>
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params,
n_estimators=1000 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
!pip install seaborn==0.11.0<import_modules>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
pd.options.display.max_rows=200
pd.set_option('mode.chained_assignment', None)
 simplefilter("ignore", category=ConvergenceWarning)
 simplefilter("ignore", category=RuntimeWarning )<load_from_csv>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
train = pd.read_csv('/kaggle/input/titanic/train.csv', index_col='PassengerId')
 test = pd.read_csv('/kaggle/input/titanic/test.csv', index_col='PassengerId' )<count_missing_values>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
train.isna().sum()<count_missing_values>
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 trn_all_predictions.to_csv('trn_all_predictions.csv', index=False)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
test.isna().sum()<categorify>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
def imputer(df):
age_impute_series = df.groupby(['Pclass', 'Sex'] ).Age.transform('mean')
 df.Age.fillna(age_impute_series, inplace=True)
 df.Cabin = df.Cabin.str.extract(pat='([A-Z])')
 df.Cabin.fillna('M', inplace=True)
 df['Deck'] = df.Cabin.replace({'A':'ABC', 'B':'ABC', 'C':'ABC', 'D':'DE', 'E':'DE', 'F':'FG',
'G':'FG', 'T':'ABC'})
 df.drop('Cabin', axis=1, inplace=True)
 for feature in df.columns:
df[feature].fillna(df[feature].mode() [0], inplace=True)
 return df<categorify>
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_all_predictions.to_csv('sub_all_predictions.csv',index = False)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
train_imputed = imputer(train.copy())
 test_imputed = imputer(test.copy() )<categorify>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
def ticket_extractor(ticket):
alpha = re.sub('\d', '', ticket)
 if alpha:
return alpha
else:
num = re.search('\d{1,9}', ticket)
 return ticket
temp = train_imputed.copy()
temp['Ticket_extracted'] = temp.Ticket.apply(ticket_extractor)
 for i in range(len(temp.Ticket)) :
try:
int(temp.Ticket_extracted.iloc[i])
 temp.Ticket_extracted.iloc[i] = f'Num_{len(temp.Ticket_extracted.iloc[i])}'
except:
continue
for label, pattern in [('_ca', 'C[.]?A[.]?'),('_PC', 'PC'),('_SOTON', 'SOTON'),('_STON', 'STON'), 
('_WC', 'W[.]?[/]?C'),('_SC', 'S[.]?C[.]?'),('_A', 'A[.]?'),('_SOC', 'S[.]?O[.]?[/]?C'), 
('_PP', 'PP'),('_FC', '(F.C.|F.C.C.) '),('_LS_number', 'Num_(6|7)'),('_SS_number', 'Num_(3|4|5)'), 
('rare', '^[^_]')]:
temp.Ticket_extracted[temp.Ticket_extracted.str.contains(pattern)] = label
temp['Ticket_extracted'].value_counts(dropna=False)
 fig, ax = plt.subplots(2, 1, figsize=(15, 15))
plt.style.use('fivethirtyeight')
 sns.countplot(data=temp, x='Ticket_extracted', hue='Survived', ax=ax[0], palette=palette_6)
 ax[0].set_title('Extracted Ticket - Survival "count" plot', size=20, loc='Left', y=1.03)
 sns.barplot(data=temp, x='Ticket_extracted', y='Survived', ax=ax[1], palette=palette_6)
 ax[1].set_title('Extracted Ticket - Survival "chance" plot', size=20, loc='Left', y=1.03)
 sns.despine()
plt.show()<feature_engineering>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
', params)
 Best num_boost_round:', len(cv_results['rmse-mean']))
temp = train_imputed.copy()
temp['Title'] = temp.Name.str.extract(pat='([a-zA-Z]+\.) ')
 temp.Title[~temp.Title.isin(['Mr.', 'Miss.', 'Mrs.', 'Master.'])] = 'rare'<categorify>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(**params,
n_estimators=1500, 
)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5
def feature_creator(df_train, df_test):
df_train['Fare_cat'] = pd.qcut(df_train['Fare'], 7)
 df_test['Fare_cat'] = pd.qcut(df_test['Fare'], 7)
 df_train['Fare_cat'] = LabelEncoder().fit_transform(df_train['Fare_cat'])
 df_test['Fare_cat'] = LabelEncoder().fit_transform(df_test['Fare_cat'])
 df_train['Age_cat'] = pd.cut(df_train.Age, bins=[0, 5, 24, 30, 36, np.inf])
 df_test['Age_cat'] = pd.cut(df_test.Age, bins=[0, 5, 24, 30, 36, np.inf])
 df_train['Age_cat'] = LabelEncoder().fit_transform(df_train['Age_cat'])
 df_test['Age_cat'] = LabelEncoder().fit_transform(df_test['Age_cat'])
 for df in [df_train, df_test]:
df['Title'] = df.Name.str.extract(pat='([a-zA-Z]+\.) ')
 df.Title[~df.Title.isin(['Mr.', 'Miss.', 'Mrs.', 'Master.'])] = 'rare'
df_full = pd.concat([df_train, df_test], axis=0)
 df_full['Ticket_frequency'] = df_full.Ticket.map(df_full.Ticket.value_counts())
 df_train = df_full.loc[df_train.index]
df_test = df_full.loc[df_test.index].loc[df_test.index].drop('Survived', axis=1)
 for df in [df_train, df_test]:
df['Ticket_extracted'] = df.Ticket.apply(ticket_extractor)
 for i in range(len(df.Ticket)) :
try:
int(df.Ticket_extracted.iloc[i])
 df.Ticket_extracted.iloc[i] = f'Num_{len(df.Ticket_extracted.iloc[i])}'
except:
continue
for label, pattern in [('_ca', 'C[.]?A[.]?'),('_PC', 'PC'),('_SOTON', 'SOTON'),('_STON', 'STON'), 
('_WC', 'W[.]?[/]?C'),('_SC', 'S[.]?C[.]?'),('_A', 'A[.]?'),('_SOC', 'S[.]?O[.]?[/]?C'), 
('_PP', 'PP'),('_FC', '(F.C.|F.C.C.) '),('_LS_number', 'Num_(6|7)'), 
('_SS_number', 'Num_(3|4|5)'),('rare', '^[^_]')]:
df.Ticket_extracted[df.Ticket_extracted.str.contains(pattern)] = label
df_train['Family_size'] = df_train['SibSp'] + df_train['Parch'] + 1
df_test['Family_size'] = df_test['SibSp'] + df_test['Parch'] + 1
df_train['Family_size_cat'] = df_train.Family_size.replace({1:'alone', 2:'small_family', 3:'small_family', 4:'small_family'
,5:'large_family', 6:'large_family', 7:'large_family'
,8:'large_family', 9:'large_family', 10:'large_family'
,11:'large_family'})
 df_test['Family_size_cat'] = df_test.Family_size.replace({1:'alone', 2:'small_family', 3:'small_family', 4:'small_family'
,5:'large_family', 6:'large_family', 7:'large_family'
,8:'large_family', 9:'large_family', 10:'large_family'
,11:'large_family'})
 df_train['Name_length'] = df_train.Name.str.replace(pat='[^a-zA-Z]', repl='' ).str.len()
df_test['Name_length'] = df_test.Name.str.replace(pat='[^a-zA-Z]', repl='' ).str.len()
df_full = pd.concat([df_train, df_test], axis=0 ).reset_index()
df_full['Family_name'] = df_full.Name.str.split(',', n=1, expand=True ).iloc[:, 0]
df_full['Family_Survival'] = 0.5
for grp, grp_df in df_full.groupby(['Family_name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in df_full.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 0
df_full.set_index('PassengerId', inplace=True)
 df_train = df_full[:891]
df_test = df_full[891:]
df_train.drop(['Family_name'], axis=1, inplace=True)
 df_test.drop(['Family_name', 'Survived'], axis=1, inplace=True)
 y_train =df_train.Survived
df_train.drop('Survived', axis=1, inplace=True)
 return df_train, y_train, df_test<data_type_conversions>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )