Kaggle competition: Titanic - Machine Learning from Disaster
df_plot = random_down_sample(
 g_X_vldt.merge(g_y_vldt, left_index=True, right_index=True),
majority_multiply=g_majority_multiply, target_col_name='is_attributed',
minority_val = 1, majority_val = 0); g()
df_plot['is_attributed'].value_counts()<train_on_grid>
train=pd.read_csv(".. /input/titanic/train.csv")
 test=pd.read_csv(".. /input/titanic/test.csv" )
def grid_search(X_train, y_train, X_vldt, y_vldt, base_model, param_grid):
estimator_lst,grid_point_lst,precision_lst,recall_lst,f1_lst,auc_lst=[],[],[],[],[],[]
for grid_point in list(ParameterGrid(param_grid)) :
print(f'----- grid_point:
{grid_point}')
 base_model = base_model.set_params(**grid_point)
 base_model = base_model.set_params(silent = True)
 base_model = base_model.set_params(verbosity = 0)
 print('update_data_balancing_param:')
 base_model = update_data_balancing_param(base_model, y_train.value_counts())
 print(f'model parameters:
{base_model.get_params() }')
 print('cat_fea: {}'.format(g_fit_params['categorical_feature']))
print('fea: {}'.format(X_train.columns))
print(type(X_train.columns))
print(X_train.columns.get_loc('app'))
cat_fea_indices = list(map(lambda col:X_train.columns.get_loc(col), g_fit_params['categorical_feature']))
fit_params_copy = g_fit_params.copy()
fit_params_copy['categorical_feature'] = cat_fea_indices
fit_params_copy['verbose'] = -1
print(f'fit_params_copy:
{fit_params_copy}')
 fitted = base_model.fit(
 X_train, y_train, eval_set = [(X_vldt, y_vldt)], **fit_params_copy); g()
y_pred = fitted.predict(X_vldt)
 grid_point_lst.append(grid_point)
 estimator_lst.append(fitted)
 precision_lst.append(precision_score(y_vldt, y_pred))
recall_lst.append(recall_score(y_vldt, y_pred))
f1_lst.append(f1_score(y_vldt, y_pred))
auc_lst.append(roc_auc_score(y_vldt, y_pred))
score_dict = pd.DataFrame(data={
'precision':precision_lst,
'recall':recall_lst,
'f1':f1_lst,
'auc':auc_lst
}, index=grid_point_lst)
 estimator_dict = pd.DataFrame(data={'estimator':estimator_lst}, index=grid_point_lst)
 return score_dict, estimator_dict
def run_grid_search(base_model_id, exp_params_id):
with timer_memory('grid_search %'.format(exp_params_id)) :
lgb_model, search_params = get_model_and_search_params(base_model_id, exp_params_id)
 score_dict, estimator_dict = grid_search(X_train, y_train, g_X_vldt, g_y_vldt, lgb_model, search_params)
 return score_dict<import_modules>
( train.isnull() ["Cabin"]==(train["Pclass"]==1)).value_counts()
import csv
import time
from csv import DictReader
from math import exp, log, sqrt
from numba import jit
import pandas as pd
from random import randint<define_variables>
( test.isnull() ["Cabin"]==(test["Pclass"]==1)).value_counts()
data_path = ".. /input/"
train = data_path+'train.csv'
train_s = data_path+'train_sample.csv'
test = data_path+'test.csv'
submission = 'submission.csv'<init_hyperparams>
train["Cabin"].fillna("NC",inplace=True)
 test["Cabin"].fillna("NC",inplace=True )
alpha = 0.011
beta = 0.000000001
L1 = 0.0000001
L2 = 0.0001<init_hyperparams>
train["Cabin"].isnull().value_counts()
D = 2 ** 26
interaction = False<define_variables>
test["Cabin"].isnull().value_counts()
epoch = 2
holdday = ''<choose_model_class>
train["Embarked"].fillna(train["Embarked"].mode() [0],inplace=True)
 test["Fare"].fillna(test["Fare"].median() ,inplace=True )
class ftrl_proximal(object):

 def __init__(self, alpha, beta, L1, L2, D):
self.alpha = alpha
self.beta = beta
self.L1 = L1
self.L2 = L2
self.D = D
self.n = [0.] * D
self.z = [0.] * D
self.w = {}
def _indices(self, x):

 yield 0
for index in x:
yield index
def predict(self, x):

 alpha = self.alpha
beta = self.beta
L1 = self.L1
L2 = self.L2
n = self.n
z = self.z
w = {}
wTx = 0.
 for i in self._indices(x):
sign = -1.if z[i] < 0 else 1.
 if sign * z[i] <= L1:
w[i] = 0.
 else:
w[i] =(sign * L1 - z[i])/(( beta + sqrt(n[i])) / alpha + L2)
 wTx += w[i]
self.w = w
return 1./(1.+ exp(-max(min(wTx, 35.) , -35.)))
def update(self, x, p, y):

 alpha = self.alpha
n = self.n
z = self.z
w = self.w
g = p - y
f = 1 if y == 0 else randint(300, 400)
 for i in self._indices(x):
sigma =(sqrt(n[i] + g * g)- sqrt(n[i])) / alpha
z[i] += f*(g - sigma * w[i])
 n[i] += f*(g * g)
<feature_engineering>
def get_title(name):
title_search=re.search('([A-Za-z]+)\.',name)
 if(title_search):
return title_search.group(1)
 return ""
train["Title"]=train["Name"].apply(get_title)
 test["Title"]=test["Name"].apply(get_title)
 train.head()
def temps(df):
date, time = df['click_time'].split(' ')
 hour = time.split(':')[0]
h=int(hour)
 df["Nuit"]=0
df["Matin"]=0
df["Apres-midi"]=0
df["Soir"]=0
if(h>=18):
df["Soir"]=1
if(h>=0 and h<8):
df["Nuit"]=1
if(h>=8 and h<12):
df["Matin"]=1
if(h>=12 and h<18):
df["Apres-midi"]=1<compute_test_metric>
train["Title"].value_counts()
learner = ftrl_proximal(alpha, beta, L1, L2, D )<predict_on_test>
train["Title"]=train["Title"].replace(['Dr','Rev','Major','Col','Capt','Sir','Don','Lady','Countess','Jonkheer'],'Super')
 train["Title"]=train["Title"].replace(['Mlle','Ms'],'Miss')
 train["Title"]=train["Title"].replace('Mme','Mrs')
 train["Title"].value_counts()
start_time = time.time()
for e in range(epoch):
for t, x, y, _ in data(train, D):
p = learner.predict(x)
 learner.update(x, p, y)
 if t%1000000 == 0:
print("ligne: %sM ; %sMin "%(int(t/1e+6), '%0.0f'%(( time.time() -start_time)/60)) )<save_to_csv>
test["Title"].value_counts()
start_time = time.time()
with open(submission, 'w')as outfile:
outfile.write('click_id,is_attributed
')
 for t, x, y, click_id in data(test, D):
p = learner.predict(x)
 outfile.write('%s,%s
' %(click_id, str(p)))
 if t%1000000 == 0:
print("Test Rows Processed: %sM ; %ss "%(int(t/1e+6), '%0.0f'%(time.time() -start_time)))
 print('Commit & Run !' )<set_options>
test["Title"]=test["Title"].replace('Ms','Miss')
 test["Title"]=test["Title"].replace(["Col","Rev","Dr","Dona"],"Super")
 test["Title"].value_counts()
pd.set_option('display.max_columns', 100 )<define_variables>
train.drop("Name",axis=1,inplace=True)
 train.head()
VALIDATE = False
VALID_SIZE = 0.90
VALIDATE_KFOLDS = True
NUMBER_KFOLDS = 5
SAMPLE = True
RANDOM_STATE = 2018
MAX_ROUNDS = 1000
EARLY_STOP = 50
OPT_ROUNDS = 650
skiprows = range(1,109903891)
 nrows = 75000000
SAMPLE_SIZE = 1
output_filename = 'submission.csv'
IS_LOCAL = False
if(IS_LOCAL):
PATH = '.. /input/talkingdata-adtracking-fraud-detection/'
else:
PATH = '.. /input/'
print(os.listdir(PATH))<load_from_csv>
test.drop("Name",axis=1,inplace=True)
 test.head()
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
train_cols = ['ip','app','device','os', 'channel', 'click_time', 'is_attributed']
if SAMPLE:
trainset = pd.read_csv(PATH+"train_sample.csv", dtype=dtypes, usecols=train_cols)
 else:
trainset = pd.read_csv(PATH+"train.csv", skiprows=skiprows, nrows=nrows,dtype=dtypes, usecols=train_cols)
 trainset = trainset.sample(frac=SAMPLE_SIZE)
 testset = pd.read_csv(PATH+"test.csv" )<sort_values>
train=pd.get_dummies(train,columns=["Sex","Title","Embarked"])
 train.head()
def missing_data(data):
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() /data.isnull().count() *100 ).sort_values(ascending = False)
 return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'] )<count_missing_values>
test=pd.get_dummies(test,columns=["Sex","Title","Embarked"])
 test.head()
missing_data(trainset )<count_missing_values>
rfr1=RandomForestRegressor()
col_age=["Title_Master","Title_Miss","Title_Mr","Title_Mrs","Title_Super","Fare","Parch","SibSp"]
x_list1=[]
x_list2=[]
for i in range(len(train)) :
if train["Age"].isnull().loc[i]==False:
x_list1.append(i)
 else:
x_list2.append(i)
 X_agetrain=train.loc[x_list1,col_age]
y_agetrain=train.loc[x_list1,"Age"]
rfr1.fit(X_agetrain,y_agetrain)
 for i in range(len(train)) :
if(train["Age"].isnull().loc[i]==True):
train.loc[x_list2,"Age"]=rfr1.predict(train.loc[x_list2,col_age] )
missing_data(testset )<feature_engineering>
train["Age"].isnull().value_counts()
trainset['year'] = pd.to_datetime(trainset.click_time ).dt.year
trainset['month'] = pd.to_datetime(trainset.click_time ).dt.month
trainset['day'] = pd.to_datetime(trainset.click_time ).dt.day
trainset['hour'] = pd.to_datetime(trainset.click_time ).dt.hour
trainset['min'] = pd.to_datetime(trainset.click_time ).dt.minute
trainset['sec'] = pd.to_datetime(trainset.click_time ).dt.second
trainset.head()<categorify>
rfr2=RandomForestRegressor()
col_age=["Title_Master","Title_Miss","Title_Mr","Title_Mrs","Title_Super","Fare","Parch","SibSp"]
x_list_1=[]
x_list_2=[]
for i in range(len(test)) :
if test["Age"].isnull().loc[i]==False:
x_list_1.append(i)
 else:
x_list_2.append(i)
 X_agetest=test.loc[x_list_1,col_age]
y_agetest=test.loc[x_list_1,"Age"]
rfr2.fit(X_agetest,y_agetest)
 for i in range(len(test)) :
if(test["Age"].isnull().loc[i]==True):
test.loc[x_list_2,"Age"]=rfr2.predict(test.loc[x_list_2,col_age] )
def show_max_clean(df,gp,agg_name,agg_type,show_max):
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 gc.collect()
return(df)
 def perform_count(df, group_cols, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Aggregating by ", group_cols , '...')
 gp = df[group_cols][group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_countuniq(df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Counting unique ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_cumcount(df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Cumulative count by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_mean(df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True):
if show_agg:
print("Calculating mean of ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_var(df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True):
if show_agg:
print("Calculating variance of ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))<feature_engineering>
test["Age"].isnull().value_counts()
testset['year'] = pd.to_datetime(testset.click_time ).dt.year
testset['month'] = pd.to_datetime(testset.click_time ).dt.month
testset['day'] = pd.to_datetime(testset.click_time ).dt.day
testset['hour'] = pd.to_datetime(testset.click_time ).dt.hour
testset['min'] = pd.to_datetime(testset.click_time ).dt.minute
testset['sec'] = pd.to_datetime(testset.click_time ).dt.second
testset.head()<count_unique_values>
train=pd.get_dummies(train,columns=["Fare_bin","Age_bin"])
 test=pd.get_dummies(test,columns=["Fare_bin","Age_bin"])
 train.head()
testset = perform_countuniq(testset, ['ip'], 'channel', 'X0', 'uint8', show_max=True); gc.collect()
testset = perform_cumcount(testset, ['ip', 'device', 'os'], 'app', 'X1', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'day'], 'hour', 'X2', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip'], 'app', 'X3', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'app'], 'os', 'X4', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip'], 'device', 'X5', 'uint16', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['app'], 'channel', 'X6', show_max=True); gc.collect()
testset = perform_cumcount(testset, ['ip'], 'os', 'X7', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'device', 'os'], 'app', 'X8', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'day', 'hour'], 'ip_tcount', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'app'], 'ip_app_count', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'app', 'os'], 'ip_app_os_count', 'uint16', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'day', 'channel'], 'hour', 'ip_tchan_count', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'app', 'os'], 'hour', 'ip_app_os_var', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'app', 'channel'], 'day', 'ip_app_channel_var_day', show_max=True); gc.collect()
testset = perform_mean(testset, ['ip', 'app', 'channel'], 'hour', 'ip_app_channel_mean_hour', show_max=True); gc.collect()<data_type_conversions>
train.drop(["Age","Fare","Ticket"],axis=1,inplace=True)
 test.drop(["Age","Fare","Ticket"],axis=1,inplace=True)
 train.head()
start = datetime.now()
len_train = len(trainset)
 gc.collect()
most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]
least_freq_hours_in_test_data = [6, 11, 15]
def prep_data(df):
df['hour'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 df['day'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 df.drop(['click_time'], axis=1, inplace=True)
 gc.collect()
df['in_test_hh'] =(3
- 2*df['hour'].isin(most_freq_hours_in_test_data)
 - 1*df['hour'].isin(least_freq_hours_in_test_data)).astype('uint8')
 gp = df[['ip', 'day', 'in_test_hh', 'channel']].groupby(by=['ip', 'day', 'in_test_hh'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_test_hh'})
 df = df.merge(gp, on=['ip','day','in_test_hh'], how='left')
 df.drop(['in_test_hh'], axis=1, inplace=True)
 df['nip_day_test_hh'] = df['nip_day_test_hh'].astype('uint32')
 del gp
gc.collect()
gp = df[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_hh'})
 df = df.merge(gp, on=['ip','day','hour'], how='left')
 df['nip_day_hh'] = df['nip_day_hh'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'os', 'hour', 'channel']].groupby(by=['ip', 'os', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_os'})
 df = df.merge(gp, on=['ip','os','hour'], how='left')
 df['nip_hh_os'] = df['nip_hh_os'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'app', 'hour', 'channel']].groupby(by=['ip', 'app', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_app'})
 df = df.merge(gp, on=['ip','app','hour'], how='left')
 df['nip_hh_app'] = df['nip_hh_app'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'device', 'hour', 'channel']].groupby(by=['ip', 'device', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_dev'})
 df = df.merge(gp, on=['ip','device','hour'], how='left')
 df['nip_hh_dev'] = df['nip_hh_dev'].astype('uint32')
 del gp
gc.collect()
df.drop(['ip','day'], axis=1, inplace=True)
 gc.collect()
return df<init_hyperparams>
print(train["Cabin"].values.tolist() )
trainset = prep_data(trainset)
 gc.collect()
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':'auc',
'learning_rate': 0.1,
'num_leaves': 9,
'max_depth': 5,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.9,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
}
target = 'is_attributed'
predictors = ['app','device','os', 'channel', 'hour', 'nip_day_test_hh', 'nip_day_hh', 'nip_hh_os', 'nip_hh_app', 'nip_hh_dev']
categorical = ['app', 'device', 'os', 'channel', 'hour'] 
<split>
train["Cabin_encoding"]=train["Pclass"]
for i in range(len(train)) :
if(train["Cabin"].loc[i]=="NC"):
train["Cabin_encoding"].loc[i]=1000
else:
if(len(train["Cabin"].loc[i])>4):
train["Cabin_encoding"].loc[i]=500
elif(len(train["Cabin"].loc[i])>1):
temp=train["Cabin"].loc[i]
train["Cabin_encoding"].loc[i]=(ord(temp[0])-ord('A')) *125+int(temp[1:])
 else:
temp=train["Cabin"].loc[i]
train["Cabin_encoding"].loc[i]=min(( ord(temp[0])-ord('A')) *125,1000)
 train[["Cabin","Cabin_encoding"]].head()
if VALIDATE:
train_df, val_df = train_test_split(trainset, test_size=VALID_SIZE, random_state=RANDOM_STATE, shuffle=True)
 dtrain = lgb.Dataset(train_df[predictors].values,
label=train_df[target].values,
feature_name=predictors,
categorical_feature=categorical)
 del train_df
gc.collect()
dvalid = lgb.Dataset(val_df[predictors].values,
label=val_df[target].values,
feature_name=predictors,
categorical_feature=categorical)
 del val_df
gc.collect()
evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=MAX_ROUNDS,
early_stopping_rounds=EARLY_STOP,
verbose_eval=50,
feval=None)
 del dvalid
elif VALIDATE_KFOLDS:
kf = KFold(n_splits = NUMBER_KFOLDS, random_state = RANDOM_STATE, shuffle = True)
 for train_index, test_index in kf.split(trainset):
train_X, valid_X = trainset.iloc[train_index], trainset.iloc[test_index]
dtrain = lgb.Dataset(train_X[predictors].values, label=train_X[target].values,
feature_name=predictors, categorical_feature=categorical)
 dvalid = lgb.Dataset(valid_X[predictors].values, label=valid_X[target].values,
feature_name=predictors, categorical_feature=categorical)
 evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=MAX_ROUNDS,
early_stopping_rounds=EARLY_STOP,
verbose_eval=50,
feval=None)
 else:
gc.collect()
dtrain = lgb.Dataset(train_df[predictors].values, label=train_df[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train_df
gc.collect()
evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain],
valid_names=['train'],
evals_result=evals_results,
num_boost_round=OPT_ROUNDS,
verbose_eval=50,
feval=None)
 del dtrain
gc.collect()<save_to_csv>
test["Cabin_encoding"]=test["Pclass"]
for i in range(len(test)) :
if(test["Cabin"].loc[i]=="NC"):
test["Cabin_encoding"].loc[i]=1000
else:
if(len(test["Cabin"].loc[i])>4):
test["Cabin_encoding"].loc[i]=500
elif(len(test["Cabin"].loc[i])>1):
temp=test["Cabin"].loc[i]
test["Cabin_encoding"].loc[i]=(ord(temp[0])-ord('A')) *125+int(temp[1:])
 else:
temp=test["Cabin"].loc[i]
test["Cabin_encoding"].loc[i]=min(( ord(temp[0])-ord('A')) *125,1000 )
test_cols = ['ip','app','device','os', 'channel', 'click_time', 'click_id']
test_df = prep_data(testset)
 gc.collect()
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id']
sub['is_attributed'] = model.predict(test_df[predictors])
 sub.to_csv(output_filename, index=False, float_format='%.9f')
<define_variables>
train.drop("Cabin",axis=1,inplace=True)
 train.head()
FILENO= 1
debug=0 
<merge>
test.drop("Cabin",axis=1,inplace=True)
 test.head()
def do_agg(df, group_cols, agg_type='uint8', show_max=False, show_agg=True):
agg_name='{}_agg'.format('_'.join(group_cols))
if show_agg:
print("
Aggregating by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<groupby>
train["Cabin_encoding"]=train["Cabin_encoding"]/1000
train.info()
def do_count(df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True):
agg_name= '{}_by_{}_count'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].count().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<count_unique_values>
train.head()
test["Cabin_encoding"]=test["Cabin_encoding"]/1000
def do_countuniq(df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True):
agg_name= '{}_by_{}_countuniq'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting unqiue ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<groupby>
X=train.drop(["Survived","PassengerId"],axis=1)
 y=train["Survived"]
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=100 )
def do_cumcount(df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_cumcount'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Cumulative count by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<categorify>
ids=test["PassengerId"]
Xtest=test.drop(["PassengerId"],axis=1)
 pred_1=lr.predict(Xtest)
 pred_2=ranF.predict(Xtest)
 pred_3=abc.predict(Xtest)
 pred_4=gbc.predict(Xtest)
 pred_5=clf.predict(Xtest)
 pred=[]
for i in range(len(pred_1)) :
l_0=1
l_1=1
if(pred_1[i]==0):
l_0*=p1[0,0]
l_1*=p1[1,0]
else:
l_0*=p1[0,1]
l_1*=p1[1,1]
if(pred_2[i]==0):
l_0*=p2[0,0]
l_1*=p2[1,0]
else:
l_0*=p2[0,1]
l_1*=p2[1,1]
if(pred_3[i]==0):
l_0*=p3[0,0]
l_1*=p3[1,0]
else:
l_0*=p3[0,1]
l_1*=p3[1,1]
if(pred_4[i]==0):
l_0*=p4[0,0]
l_1*=p4[1,0]
else:
l_0*=p4[0,1]
l_1*=p4[1,1]
if(pred_5[i]==0):
l_0*=p4[0,0]
l_1*=p4[1,0]
else:
l_0*=p4[0,1]
l_1*=p4[1,1]
if(l_0>l_1):
pred.append(0)
 else:
pred.append(1 )
def do_mean(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_mean'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating mean of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )<categorify>
Survived=pd.Series(pred_5)
 output=pd.concat([ids,Survived],axis=1)
 output.columns=["PassengerId","Survived"]
<drop_column><EOS> 
output.to_csv("submission.csv",index=False)
 output.head()