Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
sns.set_style("whitegrid")
 %matplotlib inline
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))
model_order = [m for m in base_models]<prepare_x_and_y>
training = pd.read_csv(".. /input/train.csv")
 testing = pd.read_csv(".. /input/test.csv" )
cvdata = pd.DataFrame({
m:pd.read_csv(cvfiles[m])['is_attributed'].clip(almost_zero,almost_one ).apply(logit)
 for m in base_models
})
 X_train = np.array(cvdata[model_order])
 y_train = pd.read_pickle(VAL_FILE)['is_attributed']<train_model>
def null_table(training, testing):
print("Training Data Frame")
 print(pd.isnull(training ).sum())
 print(" ")
 print("Testing Data Frame")
 print(pd.isnull(testing ).sum())
 null_table(training, testing )
stack_model = LogisticRegression()
stack_model.fit(X_train, y_train)
 stack_model.coef_<compute_test_metric>
training.drop(labels = ["Cabin", "Ticket"], axis = 1, inplace = True)
 testing.drop(labels = ["Cabin", "Ticket"], axis = 1, inplace = True)
 null_table(training, testing )
print('Stacker score: ', roc_auc_score(y_train, stack_model.predict_proba(X_train)[:,1]))<load_from_csv>
training["Age"].fillna(training["Age"].median() , inplace = True)
 testing["Age"].fillna(testing["Age"].median() , inplace = True)
 training["Embarked"].fillna("S", inplace = True)
 testing["Fare"].fillna(testing["Fare"].median() , inplace = True)
 null_table(training, testing )
final_sub = pd.DataFrame()
subs = {m:pd.read_csv(subfiles[m] ).rename({'is_attributed':m},axis=1)for m in base_models}
first_model = list(base_models.keys())[0]
final_sub['click_id'] = subs[first_model]['click_id']<merge>
le_sex = LabelEncoder()
le_sex.fit(training["Sex"])
 encoded_sex_training = le_sex.transform(training["Sex"])
 training["Sex"] = encoded_sex_training
encoded_sex_testing = le_sex.transform(testing["Sex"])
 testing["Sex"] = encoded_sex_testing
le_embarked = LabelEncoder()
le_embarked.fit(training["Embarked"])
 encoded_embarked_training = le_embarked.transform(training["Embarked"])
 training["Embarked"] = encoded_embarked_training
encoded_embarked_testing = le_embarked.transform(testing["Embarked"])
 testing["Embarked"] = encoded_embarked_testing
df = subs[first_model]
for m in base_models:
if m != first_model:
df = df.merge(subs[m], on='click_id')
 df.head()<predict_on_test>
training["FamSize"] = training["SibSp"] + training["Parch"] + 1
testing["FamSize"] = testing["SibSp"] + testing["Parch"] + 1
X_test = np.array(df.drop(['click_id'],axis=1)[model_order].clip(almost_zero,almost_one ).apply(logit))
final_sub['is_attributed'] = stack_model.predict_proba(X_test)[:,1]
final_sub.head(10 )<save_to_csv>
training["IsAlone"] = training.FamSize.apply(lambda x: 1 if x == 1 else 0)
 testing["IsAlone"] = testing.FamSize.apply(lambda x: 1 if x == 1 else 0 )
final_sub.to_csv("sub_stacked.csv", index=False, float_format='%.9f' )<define_variables>
for name in training["Name"]:
training["Title"] = training["Name"].str.extract("([A-Za-z]+)\.",expand=True)
 for name in testing["Name"]:
testing["Title"] = testing["Name"].str.extract("([A-Za-z]+)\.",expand=True )
NROWS1 = 3000000
NROWS2 = 1000000
RG = 500000
SKIPROWS = range(1,RG)
 path = '.. /input/'
path_train = path + 'train.csv'
path_test = path + 'test.csv'
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('Loading the training data...')
 train = pd.read_csv(path_train, skiprows=SKIPROWS, nrows=NROWS1, dtype=dtypes, header=0)
 print('End loading train data...')
 print('Loading the test data...')
 test = pd.read_csv(path_test, dtype=dtypes, header=0)
 print('End loading test data...' )<data_type_conversions>
titles = set(training["Title"])
 print(titles )
print("Unique data train")
 Unique_data_train = pd.to_datetime(train.click_time ).dt.day.astype('uint8' ).value_counts().sort_index()
print(Unique_data_train)
 print("Unique data test")
 Unique_data_test = pd.to_datetime(test.click_time ).dt.day.astype('uint8' ).value_counts().sort_index()
print(Unique_data_test )<train_model>
title_list = list(training["Title"])
 frequency_titles = []
for i in titles:
frequency_titles.append(title_list.count(i))
print(frequency_titles )
len_train = len(train)
 print('The initial size of the train set is', len_train)
 train = train.append(test)
 print('Binding the training and test set together...')
 del test<train_model>
titles = list(titles)
 title_dataframe = pd.DataFrame({
"Titles" : titles,
"Frequency" : frequency_titles
})
 print(title_dataframe )
print("Train and test together")
 print(train.head(5))<data_type_conversions>
title_replacements = {"Mlle": "Other", "Major": "Other", "Col": "Other", "Sir": "Other", "Don": "Other", "Mme": "Other",
"Jonkheer": "Other", "Lady": "Other", "Capt": "Other", "Countess": "Other", "Ms": "Other", "Dona": "Other"}
training.replace({"Title": title_replacements}, inplace=True)
 testing.replace({"Title": title_replacements}, inplace=True)
 le_title = LabelEncoder()
le_title.fit(training["Title"])
 encoded_title_training = le_title.transform(training["Title"])
 training["Title"] = encoded_title_training
encoded_title_testing = le_title.transform(testing["Title"])
 testing["Title"] = encoded_title_testing
train['hour'] = pd.to_datetime(train.click_time ).dt.hour.astype('uint8')
 train['day'] = pd.to_datetime(train.click_time ).dt.day.astype('uint8')
 train['wday'] = pd.to_datetime(train.click_time ).dt.dayofweek.astype('uint8')
 train['minute'] = pd.to_datetime(train.click_time ).dt.minute.astype('uint8')
 train['second'] = pd.to_datetime(train.click_time ).dt.second.astype('uint8')
 train["doy"] = pd.to_datetime(train.click_time ).dt.dayofyear.astype('uint8')
 train = train.drop(['click_time', 'attributed_time'], axis=1)
 print(train.dtypes )<count_values>
training.drop("Name", axis = 1, inplace = True)
 testing.drop("Name", axis = 1, inplace = True )
print("Frequent hours")
 frequent_hour = train.hour.value_counts().sort_index()
print(frequent_hour)
 print("Frequent days")
 frequent_day = train.day.value_counts().sort_index()
print(frequent_day)
 print("Frequent doy")
 frequent_doy = train.doy.value_counts().sort_index()
print(frequent_doy)
 print("Frequent week days")
 frequent_wday = train.wday.value_counts().sort_index()
print(frequent_wday)
 print("Frequent minutes")
 frequent_minute = train.minute.value_counts().sort_index()
print(frequent_minute)
 print("Frequent seconds")
 frequent_second = train.second.value_counts().sort_index()
print(frequent_minute )<data_type_conversions>
scaler = StandardScaler()
ages_train = np.array(training["Age"] ).reshape(-1, 1)
 fares_train = np.array(training["Fare"] ).reshape(-1, 1)
 ages_test = np.array(testing["Age"] ).reshape(-1, 1)
 fares_test = np.array(testing["Fare"] ).reshape(-1, 1)
 training["Age"] = scaler.fit_transform(ages_train)
 training["Fare"] = scaler.fit_transform(fares_train)
 testing["Age"] = scaler.fit_transform(ages_test)
 testing["Fare"] = scaler.fit_transform(fares_test)
most_freq_hours_in_data = [4, 5, 9, 10, 13, 14]
middle1_freq_hours_in_data = [16, 17, 22]
least_freq_hours_in_data = [6, 11, 15]
train['in_hh'] =(4
- 3*train['hour'].isin(most_freq_hours_in_data)
 - 2*train['hour'].isin(middle1_freq_hours_in_data)
 - 1*train['hour'].isin(least_freq_hours_in_data)).astype('uint8')
 gp = train[['ip', 'day', 'in_hh', 'channel']].groupby(by=['ip', 'day', 'in_hh'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_hh'})
 train = train.merge(gp, on=['ip','day','in_hh'], how='left')
 train.drop(['in_hh'], axis=1, inplace=True)
 train['nip_day_hh'] = train['nip_day_hh'].astype('uint32')
 del gp
gc.collect()<train_model>
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
print("Train new time parameters")
 print(train.dtypes )<data_type_conversions>
from sklearn.metrics import make_scorer, accuracy_score
train['app'] = train['app'].astype('uint16')
 train['channel'] = train['channel'].astype('uint16')
 train['device'] = train['device'].astype('uint16')
 train['ip'] = train['ip'].astype('uint32')
 train['os'] = train['os'].astype('uint16' )<data_type_conversions>
from sklearn.model_selection import GridSearchCV
train_X = train[:len_train].drop(['click_id', 'is_attributed'], axis=1)
 train_y = train[:len_train]['is_attributed'].astype('uint8')
 test_X = train[len_train:].drop(['click_id', 'is_attributed'], axis=1)
 test_id = train[len_train:]['click_id'].astype('int')
 del train<load_from_csv>
from sklearn.model_selection import GridSearchCV
predictors = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second',
'ip_app_channel_var_day',
'ip_app_device_var_day',
'ip_app_os_var_day',
'ip_day_hour_count_channel',
'ip_app_count_channel',
'ip_app_os_count_channel',
'ip_app_day_hour_count_channel',
'ip_app_channel_mean_hour',
'nip_day_hh']
categorical = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second']<init_hyperparams>
X_train = training.drop(labels=["PassengerId", "Survived"], axis=1)
 y_train = training["Survived"]
X_test = testing.drop("PassengerId", axis=1)
metrics = 'auc'
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 7,
'max_depth': 4,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
'metric':metrics
}
early_stopping_rounds = 100
num_boost_round = 10000
print("Preparing validation datasets")
 train_X, val_X = train_test_split(train_X, train_size=.95, shuffle=False)
 train_y, val_y = train_test_split(train_y, train_size=.95, shuffle=False)
 print("End preparing validation datasets")
 xgtrain = lgb.Dataset(train_X[predictors].values, label=train_y,feature_name=predictors,
categorical_feature=categorical)
 xgvalid = lgb.Dataset(val_X[predictors].values, label=val_y,feature_name=predictors,
categorical_feature=categorical)
 evals_results = {}
model_lgb = lgb.train(lgb_params,xgtrain,valid_sets=[xgtrain, xgvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10, feval=None)
<create_dataframe>
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0 )
sub = pd.DataFrame()
sub['click_id'] = test_id
print("Sub dimension " + str(sub.shape))
print("Test_X dimension " + str(test_X.shape))<save_to_csv>
svc_clf = SVC()
parameters_svc = {"kernel": ["rbf", "linear"], "probability": [True, False], "verbose": [True, False]}
grid_svc = GridSearchCV(svc_clf, parameters_svc, scoring=make_scorer(accuracy_score))
grid_svc.fit(X_training, y_training)
 svc_clf = grid_svc.best_estimator_
svc_clf.fit(X_training, y_training)
 pred_svc = svc_clf.predict(X_valid)
 acc_svc = accuracy_score(y_valid, pred_svc )
print("Predicting...")
 sub['is_attributed'] = model_lgb.predict(test_X[predictors])
 print("Writing...")
 sub.to_csv('sub_Yatsenko_01.csv',index=False)
 print("Done..." )<save_to_csv>
print("The Score for SVC is: " + str(acc_svc))
<load_from_csv>
linsvc_clf = LinearSVC()
parameters_linsvc = {"multi_class": ["ovr", "crammer_singer"], "fit_intercept": [True, False], "max_iter": [100, 500, 1000, 1500]}
grid_linsvc = GridSearchCV(linsvc_clf, parameters_linsvc, scoring=make_scorer(accuracy_score))
grid_linsvc.fit(X_training, y_training)
 linsvc_clf = grid_linsvc.best_estimator_
linsvc_clf.fit(X_training, y_training)
 pred_linsvc = linsvc_clf.predict(X_valid)
 acc_linsvc = accuracy_score(y_valid, pred_linsvc)
 print("The Score for LinearSVC is: " + str(acc_linsvc))
s = pd.read_csv('.. /input/adding-to-the-blender-lb-0-9688/average_result.csv' )<save_to_csv>
rf_clf = RandomForestClassifier()
parameters_rf = {"n_estimators": [4, 5, 6, 7, 8, 9, 10, 15], "criterion": ["gini", "entropy"], "max_features": ["auto", "sqrt", "log2"],
"max_depth": [2, 3, 5, 10], "min_samples_split": [2, 3, 5, 10]}
grid_rf = GridSearchCV(rf_clf, parameters_rf, scoring=make_scorer(accuracy_score))
grid_rf.fit(X_training, y_training)
 rf_clf = grid_rf.best_estimator_
rf_clf.fit(X_training, y_training)
 pred_rf = rf_clf.predict(X_valid)
 acc_rf = accuracy_score(y_valid, pred_rf)
 print("The Score for Random Forest is: " + str(acc_rf))
s.to_csv('submission11.csv', index=False )<define_variables>
logreg_clf = LogisticRegression()
parameters_logreg = {"penalty": ["l2"], "fit_intercept": [True, False], "solver": ["newton-cg", "lbfgs", "liblinear", "sag", "saga"],
"max_iter": [50, 100, 200], "warm_start": [True, False]}
grid_logreg = GridSearchCV(logreg_clf, parameters_logreg, scoring=make_scorer(accuracy_score))
grid_logreg.fit(X_training, y_training)
 logreg_clf = grid_logreg.best_estimator_
logreg_clf.fit(X_training, y_training)
 pred_logreg = logreg_clf.predict(X_valid)
 acc_logreg = accuracy_score(y_valid, pred_logreg)
 print("The Score for Logistic Regression is: " + str(acc_logreg))
is_valid = False<define_variables>
knn_clf = KNeighborsClassifier()
parameters_knn = {"n_neighbors": [3, 5, 10, 15], "weights": ["uniform", "distance"], "algorithm": ["auto", "ball_tree", "kd_tree"],
"leaf_size": [20, 30, 50]}
grid_knn = GridSearchCV(knn_clf, parameters_knn, scoring=make_scorer(accuracy_score))
grid_knn.fit(X_training, y_training)
 knn_clf = grid_knn.best_estimator_
knn_clf.fit(X_training, y_training)
 pred_knn = knn_clf.predict(X_valid)
 acc_knn = accuracy_score(y_valid, pred_knn)
 print("The Score for KNeighbors is: " + str(acc_knn))
start_time = time.time()
train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id']
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
train=pd.read_csv(".. /input/train.csv", skiprows=range(1,129903891), nrows=61000000, usecols=train_columns, dtype=dtypes)
 test=pd.read_csv(".. /input/test_supplement.csv",usecols=test_columns, dtype=dtypes)
 print('loading of data is completed in [{}] seconds'.format(time.time() - start_time))<data_type_conversions>
gnb_clf = GaussianNB()
parameters_gnb = {}
grid_gnb = GridSearchCV(gnb_clf, parameters_gnb, scoring=make_scorer(accuracy_score))
grid_gnb.fit(X_training, y_training)
 gnb_clf = grid_gnb.best_estimator_
gnb_clf.fit(X_training, y_training)
 pred_gnb = gnb_clf.predict(X_valid)
 acc_gnb = accuracy_score(y_valid, pred_gnb)
 print("The Score for Gaussian NB is: " + str(acc_gnb))
def datatimeFeatures(df):
df['datetime'] = pd.to_datetime(df['click_time'])
 df['dow'] = df['datetime'].dt.dayofweek
df['doy'] = df['datetime'].dt.dayofyear
df.drop(['click_time', 'datetime'], axis=1, inplace=True)
 return df<prepare_x_and_y>
dt_clf = DecisionTreeClassifier()
parameters_dt = {"criterion": ["gini", "entropy"], "splitter": ["best", "random"], "max_features": ["auto", "sqrt", "log2"]}
grid_dt = GridSearchCV(dt_clf, parameters_dt, scoring=make_scorer(accuracy_score))
grid_dt.fit(X_training, y_training)
 dt_clf = grid_dt.best_estimator_
dt_clf.fit(X_training, y_training)
 pred_dt = dt_clf.predict(X_valid)
 acc_dt = accuracy_score(y_valid, pred_dt)
 print("The Score for Decision Tree is: " + str(acc_dt))
y = train['is_attributed']
train.drop(['is_attributed'], axis=1, inplace=True)
 sub = pd.DataFrame()
test.drop('click_id', axis=1, inplace=True)
 gc.collect()
nrow_train = train.shape[0]
merge = pd.concat([train, test])
 del train, test
gc.collect()
ip_count = merge.groupby(['ip'])['channel'].count().reset_index()
ip_count.columns = ['ip', 'clicks_by_ip']
merge = pd.merge(merge, ip_count, on='ip', how='left', sort=False)
 merge['clicks_by_ip'] = merge['clicks_by_ip'].astype('uint16')
 merge.drop('ip', axis=1, inplace=True)
 train = merge[:nrow_train]
test = merge[nrow_train:]
del test, merge
gc.collect()
train = datatimeFeatures(train)
 gc.collect()
print('preprocessing is completed in [{}] seconds'.format(time.time() - start_time))<init_hyperparams>
xg_clf = XGBClassifier()
parameters_xg = {"objective" : ["reg:linear"], "n_estimators" : [5, 10, 15, 20]}
grid_xg = GridSearchCV(xg_clf, parameters_xg, scoring=make_scorer(accuracy_score))
grid_xg.fit(X_training, y_training)
 xg_clf = grid_xg.best_estimator_
xg_clf.fit(X_training, y_training)
 pred_xg = xg_clf.predict(X_valid)
 acc_xg = accuracy_score(y_valid, pred_xg)
 print("The Score for XGBoost is: " + str(acc_xg))
params = {'eta': 0.3,
'tree_method': "hist",
'grow_policy': "lossguide",
'max_leaves': 1400,
'max_depth': 0,
'subsample': 0.9,
'colsample_bytree': 0.7,
'colsample_bylevel':0.7,
'min_child_weight':0,
'alpha':4,
'objective': 'binary:logistic',
'scale_pos_weight':9,
'eval_metric': 'auc',
'nthread':8,
'random_state': 99,
'silent': True}<train_model>
model_performance = pd.DataFrame({
"Model": ["SVC", "Linear SVC", "Random Forest",
"Logistic Regression", "K Nearest Neighbors", "Gaussian Naive Bayes",
"Decision Tree", "XGBClassifier"],
"Accuracy": [acc_svc, acc_linsvc, acc_rf,
acc_logreg, acc_knn, acc_gnb, acc_dt, acc_xg]
})
 model_performance.sort_values(by="Accuracy", ascending=False )
if(is_valid == True):
x1, x2, y1, y2 = train_test_split(train, y, test_size=0.1, random_state=99)
 dtrain = xgb.DMatrix(x1, y1)
 dvalid = xgb.DMatrix(x2, y2)
 del x1, y2, x2, y2
gc.collect()
watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
model = xgb.train(params, dtrain, 200, watchlist, maximize=True, early_stopping_rounds = 25, verbose_eval=5)
 del dvalid
else:
dtrain = xgb.DMatrix(train, y)
 del train, y
gc.collect()
watchlist = [(dtrain, 'train')]
model = xgb.train(params, dtrain, 30, watchlist, maximize=True, verbose_eval=1)
 del dtrain
gc.collect()
print('XGBoost Training is finished in [{}] seconds'.format(time.time() - start_time))<load_from_csv>
svc_clf.fit(X_train, y_train )
test = pd.read_csv(".. /input/test.csv", usecols=test_columns, dtype=dtypes)
 test = pd.merge(test, ip_count, on='ip', how='left', sort=False)
 del ip_count
gc.collect()<data_type_conversions>
submission_predictions = svc_clf.predict(X_test )
sub['click_id'] = test['click_id'].astype('int' )<data_type_conversions>
submission = pd.DataFrame({
"PassengerId": testing["PassengerId"],
"Survived": submission_predictions
})
 submission.to_csv("titanic.csv", index=False)
 print(submission.shape )