Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
warnings.filterwarnings("ignore" )
competition_test_data = pd.read_csv('.. /input/talkingdata-adtracking-fraud-detection/test.csv', parse_dates=['click_time'] )<data_type_conversions>
df = pd.read_csv('.. /input/train.csv')
 df.head()
clicks = click_data.copy()
clicks['day'] = clicks['click_time'].dt.day.astype('uint8')
 clicks['hour'] = clicks['click_time'].dt.hour.astype('uint8')
 clicks['minute'] = clicks['click_time'].dt.minute.astype('uint8')
 clicks['second'] = clicks['click_time'].dt.second.astype('uint8' )<data_type_conversions>
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()
competition_test_data = competition_test_data.copy()
competition_test_data['day'] = competition_test_data['click_time'].dt.day.astype('uint8')
 competition_test_data['hour'] = competition_test_data['click_time'].dt.hour.astype('uint8')
 competition_test_data['minute'] = competition_test_data['click_time'].dt.minute.astype('uint8')
 competition_test_data['second'] = competition_test_data['click_time'].dt.second.astype('uint8' )<categorify>
test_df.drop(['PassengerId'],axis=1,inplace=True)
 test_df.head()
<categorify>
concated_df = pd.concat([train_df,test_df])
 concated_df.head()
unknown_value = -1
cat_features = ['ip', 'app', 'device', 'os', 'channel']
for feature in cat_features:
encoder = preprocessing.LabelEncoder()
encoder.fit(clicks[feature])
 le_dict = dict(zip(encoder.classes_, encoder.transform(encoder.classes_)))
 encoded = clicks[feature].apply(lambda x: le_dict.get(x, unknown_value))
clicks[feature+'_labels'] = encoded
competition_encoded = competition_test_data[feature].apply(lambda x: le_dict.get(x, unknown_value))
competition_test_data[feature+'_labels'] = competition_encoded<filter>
from sklearn import preprocessing as prep
train_ip_labels_unknowns = sum(clicks['ip_labels'] == unknown_value)
 train_ip_labels_unknowns<filter>
le = prep.LabelEncoder()
concated_df.Sex =le.fit_transform(concated_df.Sex)
 df.Sex[0:10]
compet_test_ip_labels_unknowns = sum(competition_test_data['ip_labels'] == unknown_value)
 compet_test_ip_labels_unknowns<define_variables>
concated_df.drop(['Cabin'],axis=1,inplace=True)
 concated_df.head()
my_own_metrics={'limit': min(limit, clicks.shape[0]),
'competition_test_data':competition_test_data.shape[0],
'train ip_labels unknowns': train_ip_labels_unknowns,
'compet_test ip_labels unknowns':compet_test_ip_labels_unknowns}
my_own_metrics<sort_values>
NameSplit = concated_df.Name.str.split('[,.]')
 NameSplit.head()
feature_cols = ['day', 'hour', 'minute', 'second',
'ip_labels', 'app_labels', 'device_labels',
'os_labels', 'channel_labels']
valid_fraction = 0.1
clicks_srt = clicks.sort_values('click_time')
 valid_rows = int(len(clicks_srt)* valid_fraction)
 train = clicks_srt[:-valid_rows * 2]
valid = clicks_srt[-valid_rows * 2:-valid_rows]
test = clicks_srt[-valid_rows:]<create_dataframe>
titles = [str.strip(name[1])for name in NameSplit.values]
titles[:10]
dtrain = lgb.Dataset(train[feature_cols], label=train['is_attributed'])
 dvalid = lgb.Dataset(valid[feature_cols], label=valid['is_attributed'])
 dtest = lgb.Dataset(test[feature_cols], label=test['is_attributed'])
 param = {'num_leaves': 64, 'objective': 'binary'}
param['metric'] = 'auc'
num_round = 1000 
<train_model>
concated_df['Title'] = titles
concated_df.head()
validation_metrics = {}
bst = lgb.train(param,
dtrain,
num_round,
valid_sets=[dvalid],
early_stopping_rounds=10,
evals_result=validation_metrics,
verbose_eval=10 )<count_values>
concated_df.Title.values[concated_df.Title.isin(['Mme', 'Mmle'])] = 'Mmle'
bst.num_trees()<compute_train_metric>
concated_df.Title.values[concated_df.Title.isin(['Capt', 'Don', 'Major', 'Sir'])] = 'Sir'
concated_df.Title.values[concated_df.Title.isin(['Dona', 'Lady', 'the Countess', 'Jonkheer'])] = 'Lady'
ypred = bst.predict(test[feature_cols])
 score = metrics.roc_auc_score(test['is_attributed'], ypred)
 print(f"Test score: {score}" )<feature_engineering>
concated_df.Title = le.fit_transform(concated_df.Title)
 concated_df.head()
my_own_metrics['test score'] = score<define_variables>
concated_df['FamilySize'] = concated_df.SibSp.values + concated_df.Parch.values + 1
feature_cols + ['click_id']<create_dataframe>
concated_df['Surname'] = surnames
concated_df['FamilyID'] = concated_df.Surname.str.cat(concated_df.FamilySize.astype(str),sep='')
 concated_df.head()
competition_test_data = competition_test_data[feature_cols + ['click_id']]<predict_on_test>
concated_df.FamilyID.values[concated_df.FamilySize.values <= 2] = 'Small'
concated_df.head()
competition_predictions = bst.predict(competition_test_data[feature_cols] )<create_dataframe>
concated_df.FamilyID.value_counts()
competition_predictions_df = pd.DataFrame(competition_predictions, columns=['is_attributed'])
 competition_predictions_df<prepare_output>
freq = list(dict(zip(concated_df.FamilyID.value_counts().index.tolist() , concated_df.FamilyID.value_counts().values)).items())
 type(freq )
competition_predictions_df['click_id'] = competition_test_data['click_id']
competition_predictions_df = competition_predictions_df[['click_id', 'is_attributed']]
competition_predictions_df<count_values>
freq[freq[:,1].astype(int)<= 2].shape
competition_predictions_df['is_attributed'].value_counts().sort_index()<save_to_csv>
freq = freq[freq[:,1].astype(int)<= 2]
competition_predictions_df.to_csv('submission.csv', index=False )<find_best_params>
concated_df.FamilyID.values[concated_df.FamilyID.isin(freq[:,0])] = 'Small'
concated_df.FamilyID.value_counts()
my_own_metrics['private score'] = 0.83173
my_own_metrics['public score'] = 0.82499
my_own_metrics<import_modules>
concated_reduce['Age'].fillna(concated_reduce['Age'].median() , inplace=True)
 concated_reduce['Fare'].fillna(concated_reduce['Fare'].median() , inplace=True )
import os
import gc
from operator import methodcaller
import numpy as np
import pandas as pd
from sklearn.preprocessing import RobustScaler as RobustScaler
from scipy.stats import skew, norm
from scipy.stats import boxcox_normmax, boxcox
from scipy.special import boxcox1p
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
import matplotlib.pyplot as plt
import seaborn as sns 
<set_options>
train_final = concated_reduce.iloc[:891].copy()
test_final = concated_reduce.iloc[891:].copy()
g_enable_log = True
def log(log_str):
if g_enable_log:
print(log_str)
 def g() :
return gc.collect()
def delete(*obj_list):
for obj in obj_list:
del obj
gc_cnt = g()
if gc_cnt > 0:
log("unreachable_obj_found: {}".format(gc_cnt))
def init_robust_boxcox() :
def robust_boxcox(data:pd.Series, lmbda=None):
if lmbda is None:
transformed, lmbda = boxcox(1 + data)
 if lmbda <= 0:
lmbda = 0
transformed = np.log1p(data)
 return(transformed, lmbda)
 else:
if lmbda <= 0:
lmbda = 0
transformed = np.log1p(data)
 return(transformed, lmbda)
 else:
transformed = boxcox(1 + data, lmbda)
 return(transformed, lmbda)
 def robust_inv_boxcox(data:pd.Series, lmbda):
if lmbda <= 0:
return np.expm1(data)
 else:
transformed = inv_boxcox(data, lmbda)- 1
return transformed
return robust_boxcox, robust_inv_boxcox
robust_boxcox, robust_inv_boxcox = init_robust_boxcox()
@contextmanager
def timer_memory(name):
t0 = time.time()
yield
print(f'Memory: {(psutil.Process(os.getpid() ).memory_info().rss/2**30):.02f}GB')
 print(f'{name} done in {time.time() -t0:.0f}s' )<load_from_csv>
X = train_final.values
X
<categorify>
y = surv_col.values
y
g_scaler_dict = {}
g_boxcox_lmbda_dict = {}
def box_cox_trans(df, fea_name, sv_policy):

 df[fea_name] = df[fea_name].astype('float64'); g()
df.loc[df[fea_name] <= 0,(fea_name)] = 0.000001
if sv_policy == 'new_and_save':
df[fea_name], lmbda = robust_boxcox(df[fea_name]); g()
g_boxcox_lmbda_dict[fea_name] = lmbda
elif sv_policy == 'reuse':
lmbda = g_boxcox_lmbda_dict[fea_name]
df[fea_name], lmbda = robust_boxcox(df[fea_name], lmbda); g()
else:
df[fea_name], lmbda = robust_boxcox(df[fea_name]); g()
log("\tboxcox lmbda: {}".format(lmbda))
return df[fea_name].astype('float32')
 def scaler_trans(df, fea_name, sv_policy, scaler):

 df[fea_name] = df[fea_name].astype('float64'); g()
if sv_policy == 'new_and_save':
df[[fea_name]] = scaler.fit_transform(df[[fea_name]]); g()
g_scaler_dict[fea_name] = scaler
elif sv_policy == 'reuse':
df[[fea_name]] = g_scaler_dict[fea_name].transform(df[[fea_name]]); g()
else:
df[[fea_name]] = scaler.fit_transform(df[[fea_name]]); g()
return df[fea_name].astype('float32' )<normalization>
test_data = test_final.values
test_data
def add_features(df, is_boxcox=False, is_scaler=False, save_transformer='reuse'):

 sv = save_transformer
bc = is_boxcox
s = None;
if is_scaler:
s = RobustScaler()
df = add_grp_nxt_clk_intv(df, ['ip','os','device','app'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['dd','hh','app','channel'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip','device','os'], 'app', 'cumcount', 'uint32', bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','device'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_stat(df, ['ip'], 'app', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_nxt_clk_intv(df, ['ip','channel'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','hh','device'], bc=bc,scl=s,sv=sv);g()
df = add_grp_nxt_clk_intv(df, ['ip','app','channel'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['app','channel'], bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','app'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip','hh'], 'app', 'nunique','uint16',bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','hh','app'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['app'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_stat(df, ['hh','app'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
return df<define_variables>
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
g_categorical_features = ['app', 'device', 'os', 'channel', 'hh']
g_non_train_columns = ['click_time', 'dd', 'ip']
def get_file_spec(is_test_file):
dtypes = {'ip':'uint32','app':'uint16','device':'uint8','os':'uint16',
'channel':'uint16','is_attributed':'int8','click_id':'int32'}
date_columns = ['click_time']
test_file_columns = ['click_time','ip','app','device','os','channel','click_id']
train_file_columns = ['click_time','ip','app','device','os','channel','is_attributed']
if is_test_file:
return dtypes, date_columns, test_file_columns
else:
return dtypes, date_columns, train_file_columns<load_from_csv>
model = Sequential()
model.add(Dense(32, init = 'uniform', activation='relu', input_dim = 10))
model.add(Dense(64, init = 'uniform', activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64, init = 'uniform', activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(12, init = 'uniform', activation='relu'))
model.add(Dense(1, init = 'uniform', activation='sigmoid'))
model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'] )
def read_data_file(file_path, is_test_file):
print('read file [is_test_file={}]: {}'.format(is_test_file, file_path))
dtypes, date_columns, file_columns = get_file_spec(is_test_file)
 df = pd.read_csv(file_path, parse_dates=date_columns,usecols=file_columns,dtype=dtypes)
 df['dd'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 df['hh'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 g()
return df
def process_ip_bucket(ip_bucket, is_down_sample, majority_multiply, tsfm_sv_policy):
print('---------- process bucket: {} -----------'.format(ip_bucket))
train_file_path = '/kaggle/input/train/train_{}.csv'.format(ip_bucket)
 test_file_path = '/kaggle/input/test/test_{}.csv'.format(ip_bucket)
 df_test = read_data_file(test_file_path, is_test_file=True)
 df_train_vldt = read_data_file(train_file_path, is_test_file=False)
 print('read files: {}; {}'.format(train_file_path, test_file_path))
df_train_vldt['click_id'] = -1
df_test['is_attributed'] = -1
df_full = pd.concat([df_train_vldt, df_test], sort=False)
 train_vldt_len = len(df_train_vldt)
 delete(df_train_vldt, df_test)
 ', df_full.head(n=5000000 ).nunique()); g()
print('add features: ')
 df_full = add_features(df_full, save_transformer=tsfm_sv_policy); g()
print('drop non-training-columns: {}'.format(g_non_train_columns))
df_full.drop(g_non_train_columns, axis=1, inplace=True); g()
print('split data set: ')
 vldt_len = min(g_vldt_set_size // g_ip_bkt_num, train_vldt_len // 5)
 df_train = df_full[: train_vldt_len - vldt_len]
df_vldt = df_full[train_vldt_len - vldt_len : train_vldt_len]
df_test = df_full[train_vldt_len :]
print('drop temporary columns: ')
 print('\tdf_train['click_id'].value_counts() ={}'.format(df_train['click_id'].value_counts()))
print('\tdf_vldt['click_id'].value_counts() ={}'.format(df_vldt['click_id'].value_counts()))
print('\tdf_test['is_attributed'].value_counts() ={}'.format(df_test['is_attributed'].value_counts()))
df_train.drop(['click_id'], axis=1, inplace=True)
 df_vldt.drop(['click_id'], axis=1, inplace=True)
 df_test.drop(['is_attributed'], axis=1, inplace=True)
 print('shape: vldt={}; test={}'.format(df_vldt.shape, df_test.shape))
print('shape: train(before downsample)={}'.format(df_train.shape))
g()
if is_down_sample == True:
print('down_sample: majority_multiply={}'.format(majority_multiply))
df_train = random_down_sample(df_train, majority_multiply)
 print('gc.collect(warnning is OK): ')
 g()
print('shape(after downsample)={}'.format(df_train.shape))
return df_train, df_vldt, df_test<categorify>
model.fit(X,y, epochs=500, batch_size = 64, verbose = 1 )
def prep_data_set_full_data() :
log_template="append bkt {}: train.shape={}; vldt.shape={}; test.shape={}"
df_train, df_vldt, df_test = process_ip_bucket(
 ip_bucket=0, is_down_sample=g_is_down_sample,
majority_multiply=g_majority_multiply,
tsfm_sv_policy='new_and_save')
 print(log_template.format(0, df_train.shape, df_vldt.shape, df_test.shape))
for bkt_id in range(1,g_ip_bkt_num):
train_bkt, vldt_bkt, test_bkt = process_ip_bucket(
 ip_bucket=bkt_id, is_down_sample=g_is_down_sample,
majority_multiply=g_majority_multiply,
tsfm_sv_policy='reuse'); g()
df_train = df_train.append(train_bkt, ignore_index=True); g()
df_vldt = df_vldt.append(vldt_bkt, ignore_index=True); g()
df_test = df_test.append(test_bkt, ignore_index=True); g()
delete(train_bkt, vldt_bkt, test_bkt)
 print(log_template.format(bkt_id, df_train.shape, df_vldt.shape, df_test.shape))
df_test.set_index('click_id', drop=True, inplace=True)
 df_test.sort_index(axis=0, inplace=True)
 return df_train, df_vldt, df_test
def feature_target_split(df, target_col_name, inplace=False):
y = df[target_col_name]; g()
if True == inplace:
df.drop(target_col_name, axis=1, inplace=True); g()
X = df
else:
X = df.drop(target_col_name, axis=1, inplace=True); g()
return X, y
def prep_feature_target_full_data() :
df_train, df_vldt, df_test = prep_data_set_full_data()
X_train, y_train = feature_target_split(df_train, target_col_name = 'is_attributed', inplace = True)
 X_vldt, y_vldt = feature_target_split(df_vldt, target_col_name = 'is_attributed', inplace = True)
 delete(df_train, df_vldt)
 print('----: prep_feature_target_full_data :-----')
 print('-- X_train={}; y_train={}; X_vldt={}; y_vldt={}; df_test={}'.format(
 X_train.shape, y_train.shape, X_vldt.shape, y_vldt.shape, df_test.shape))
print('-- features :', X_train.columns.values.tolist())
 print('-- categorical :', g_fit_params['categorical_feature'])
 print('-- y_train.value_counts
:', y_train.value_counts())
 print('-- y_vldt.value_countsl
:', y_vldt.value_counts())
 return X_train, y_train, X_vldt, y_vldt, df_test<choose_model_class>
pred = model.predict(test_data )
def default_model() :
lgb_default = LGBMClassifier()
return lgb_default.set_params(
 objective = 'binary',
metric = 'auc',
boosting_type = 'gbdt',
verbose = 1,
nthread = 4,
iid = False,
two_round = True 
)
 def gbtd_base_001() :
return default_model().set_params(
 subsample = 0.8,
subsample_freq = 1,
subsample_for_bin = 200000,
colsample_bytree = 0.8,
learning_rate = 0.08,
num_leaves = 105,
max_depth = 7,
min_split_gain = 0.3,
max_bin = 255,
reg_alpha = 0.3,
n_estimators = 2500 
)
 g_base_models = {
'gbdt_base_001' : gbtd_base_001()
}
g_search_params = {
'gbdt_base_001_exp_002' : {'min_sum_hessian_in_leaf':[0.001, 0.01, 0.05, 0.1]},
'gbdt_base_001_exp_005' : {'min_split_gain':[0.3, 0.4, 0.5]},
'gbdt_base_001_exp_010' : {'num_leaves':[90,105], 'min_split_gain':[0.3,0.4]}
}
def update_data_balancing_param(lgb_model, y_value_counts, majority_val=0, minority_val=1):
print('y_value_counts:
{}'.format(y_value_counts))
if g_scale_pos_weight is None:
print(f'use global config: scale_pos_weight={g_scale_pos_weight}')
 lgb_model.set_params(scale_pos_weight = g_scale_pos_weight)
 else :
unbalance_degree = y_value_counts[majority_val] / y_value_counts[minority_val]
print(f'majority_count/minority_count={unbalance_degree}')
 if unbalance_degree > 1.5 or unbalance_degree < 0.66:
print(f'unbalance_dgree')
 lgb_model.set_params(is_unbalance = True)
 return lgb_model
def get_model_and_search_params(base_model_id, search_params_id):
return g_base_models[base_model_id], g_search_params[search_params_id]
', search_params )<train_model>
outputBin = np.zeros(0)
 for i in pred:
if i <=.5:
outputBin = np.append(outputBin, 0)
 else:
outputBin = np.append(outputBin, 1)
 output = np.array(outputBin ).astype(int )
g_fit_params = {
'categorical_feature' : g_categorical_features,
'early_stopping_rounds' : 25,
'verbose' : 10,
'eval_metric' : 'auc'
}<train_model>
d = {'PassengerId':pessengerId, 'Survived':output}
g_base_model = g_base_models['gbdt_base_001']<split>
final_df = pd.DataFrame(data=d )
with timer_memory('prep_feature_target_full_data'):
X_train, y_train, g_X_vldt, g_y_vldt, g_df_test = prep_feature_target_full_data()<train_model>
final = final_df.to_csv('new_result.csv',index=False)
 final
with timer_memory('fit_model'):
g_model_fitted = fit_model(X_train, y_train, g_X_vldt, g_y_vldt, g_base_model )<save_to_csv>
rf = RandomForestClassifier(n_estimators=350, max_depth=15, random_state=42)
 print("train accuracy: {} ".format(rf.fit(X, y ).score(X, y)))
def predict_and_submit(model_fitted, num_iteration):
sub = pd.DataFrame()
sub['click_id'] = g_df_test.index
sub['click_id'] = sub['click_id'].astype('int')
 pred_prob = model_fitted.predict_proba(X=g_df_test, num_iteration=num_iteration)
 sub['is_attributed'] = pred_prob[:,1].reshape(-1,1)
 sub.to_csv('submit.csv', index=False, float_format='%.9f')
 print(f'num_iteration: {num_iteration}')
 print(f'sub.shape: {sub.shape}')
 return sub<predict_on_test>
rf_pred = rf.predict(test_data )
with timer_memory('predict_and_sumit'):
submit = predict_and_submit(g_model_fitted, g_model_fitted.best_iteration_)
 submit.head()<predict_on_test>
r = {'PassengerId':pessengerId, 'Survived':rf_pred}
g_y_pred_proba_vldt = g_model_fitted.predict_proba(X=g_X_vldt, num_iteration=g_model_fitted.best_iteration_)[:,1]
g_y_pred_label_vldt = g_model_fitted.predict(X=g_X_vldt, num_iteration=g_model_fitted.best_iteration_)
 g_y_vldt.value_counts()<compute_train_metric>
final_rf = pd.DataFrame(data=r )
report = classification_report(g_y_vldt, g_y_pred_label_vldt, target_names=['is_not_attributed','is_attributed'])
 print(report )<merge>
final_rf = final_df.to_csv('random_forest_result.csv',index=False)
 final_rf