Kaggle competition: Titanic - Machine Learning from Disaster
sub_df = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("lgb_submission.csv", index=False )<set_options>
%matplotlib inline
warnings.filterwarnings('ignore' )
warnings.filterwarnings('ignore' )<load_from_csv>
train =pd.read_csv(".. /input/titanic/train.csv")
 test =pd.read_csv(".. /input/titanic/test.csv")
 train.describe(include='all' )
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )<count_missing_values>
train.isnull().sum()
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
train.isnull().sum()
%%time
missing_data(train_df )<count_missing_values>
print(train.Embarked [train.Embarked == 'S'].count())
 print(train.Embarked [train.Embarked == 'C'].count())
 print(train.Embarked [train.Embarked == 'Q'].count() )
%%time
missing_data(test_df )<count_values>
train = train.fillna({"Embarked": "S"})
 embarked_mapping = {"S": 1, "C": 2, "Q": 3}
train['Embarked'] = train['Embarked'].map(embarked_mapping)
 test['Embarked'] = test['Embarked'].map(embarked_mapping)
 train.head()
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))<sort_values>
train['Sex'] = train['Sex'].map({"male": 0, "female": 1})
 test['Sex'] = test['Sex'].map({"male": 0, "female": 1})
 train.head()
%%time
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )<count_unique_values>
train = train.drop(['Cabin'], axis = 1)
 test = test.drop(['Cabin'], axis = 1 )
%%time
features = train_df.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train_df[feature].value_counts()
unique_max_train.append([feature, values.max() , values.idxmax() ])
 values = test_df[feature].value_counts()
unique_max_test.append([feature, values.max() , values.idxmax() ] )<feature_engineering>
train = train.drop(['Ticket'], axis = 1)
 test = test.drop(['Ticket'], axis = 1 )
%%time
idx = features = train_df.columns.values[2:202]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )<train_model>
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )<define_variables>
train.Age.fillna(value=train.Age.mean() , inplace=True)
 train.Fare.fillna(value=train.Fare.mean() , inplace=True)
 test.Age.fillna(value=test.Age.mean() , inplace=True)
 test.Fare.fillna(value=test.Fare.mean() , inplace=True )
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']<init_hyperparams>
train['CabinBool'] = train['CabinBool'].map({True: 0, False: 1})
 test['CabinBool'] = test['CabinBool'].map({True: 0, False: 1} )
param = {
'tree_method': 'gpu_hist',
'objective': 'binary:logitraw',
'eta':0.01,
'gamma':0.01,
'max_depth':10,
'min_child_weight':20,
'subsample':0.05,
'max_leaves':20,
'eval_metric':'auc',
'verbosity':1
}<train_model>
age_mapping = {'Baby': 1, 'Child': 2, 'Teenager': 3, 'Student': 4, 'Young': 5, 'Adult': 6, 'Senior': 7}
train['AgeGroup'] = train['AgeGroup'].map(age_mapping)
 test['AgeGroup'] = test['AgeGroup'].map(age_mapping )
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=44000)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = xgb.DMatrix(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = xgb.DMatrix(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = xgb.train(param, trn_data, num_round, evals = [(trn_data, 'train'),(val_data, 'valid')], early_stopping_rounds=3000, verbose_eval=1000)
 train_data =xgb.DMatrix(train_df.iloc[val_idx][features])
 oof[val_idx] = clf.predict(train_data, ntree_limit=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([fold_importance_df], axis=0)
 test_data = xgb.DMatrix(test_df[features])
 predictions += clf.predict(test_data, ntree_limit=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
train=train.drop(['Age'],axis =1)
 test =test.drop(['Age'],axis=1)
 train.head()
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
train.isnull().sum()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
train.isnull().sum()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
X = train.drop(['Survived', 'PassengerId'], axis=1)
 y = train["Survived"]
x_train, x_val, y_train, y_val = train_test_split(X, y)
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<feature_engineering>
log_model = LogisticRegression()
log_model.fit(x_train, y_train)
 y_pred = log_model.predict(x_val)
 acc_log=accuracy_score(y_pred, y_val)* 100
print(acc_log )
%time
idx = features = train.columns.values[2:202]
for i,df in enumerate([train, test]):
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 print('Creating percentiles features for df: {}/{}'.format(i+1,2))
df['perc_5'] = df[idx].apply(lambda x: np.percentile(x, 5), axis=1)
 df['perc_10'] = df[idx].apply(lambda x: np.percentile(x, 10), axis=1)
 df['perc_25'] = df[idx].apply(lambda x: np.percentile(x, 25), axis=1)
 df['perc_50'] = df[idx].apply(lambda x: np.percentile(x, 50), axis=1)
 df['perc_75'] = df[idx].apply(lambda x: np.percentile(x, 75), axis=1)
 df['perc_95'] = df[idx].apply(lambda x: np.percentile(x, 95), axis=1)
 df['perc_99'] = df[idx].apply(lambda x: np.percentile(x, 99), axis=1 )<prepare_x_and_y>
svm_model =SVC()
svm_model.fit(x_train,y_train)
 y_pred =svm_model.predict(x_val)
 acc_svc =accuracy_score(y_pred,y_val)*100
print(acc_svc)
X = train.iloc[:,2:].values
y = train.iloc[:,1].values
test = test.iloc[:,1:].values<train_model>
decisiontree_model =DecisionTreeClassifier()
decisiontree_model.fit(x_train,y_train)
 y_pred =decisiontree_model.predict(x_val)
 acc_decisiontree_model=accuracy_score(y_pred, y_val)*100
print(acc_decisiontree_model )
lgb.train()<create_dataframe>
randomforest_model = RandomForestClassifier()
randomforest_model.fit(x_train, y_train)
 y_pred = randomforest_model.predict(x_val)
 acc_randomforest =accuracy_score(y_pred, y_val)* 100
print(acc_randomforest )
pred = pd.DataFrame()
for i in range(1, 5):
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1,
'seed': i,
'feature_fraction_seed': i,
'bagging_seed': i,
'drop_seed': i,
'data_random_seed': i,
}
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = i)
 print(y_train)
 N = 5
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train, y_train)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_test, label=y_test)
 clf = lgb.train(param, trn_data , 10000, valid_sets = [trn_data, val_data],
verbose_eval=1000, early_stopping_rounds = 100)
 p_valid += clf.predict(X_test)
 yp += clf.predict(test)
 print(yp/N)
 pred[i] = yp/N<define_variables>
knn_model = KNeighborsClassifier()
knn_model.fit(x_train, y_train)
 y_pred = knn_model.predict(x_val)
 acc_knn_model = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(acc_knn_model )
filename = 'subm_{}_{}_'.format(ver, datetime.now().strftime('%Y-%m-%d'))
filename<save_to_csv>
sgd_model = SGDClassifier()
sgd_model.fit(x_train, y_train)
 y_pred = sgd_model.predict(x_val)
 acc_sgd_model = accuracy_score(y_pred, y_val)* 100
print(acc_sgd_model )
submission_ = pd.read_csv('.. /input/sample_submission.csv')
 submission_['target'] = pred.mean(axis=1)
 submission_.to_csv(filename+'_blend.csv', index=False )<load_from_csv>
gbk_model = GradientBoostingClassifier()
gbk_model.fit(x_train, y_train)
 y_pred = gbk_model.predict(x_val)
 acc_gbk_model= accuracy_score(y_pred, y_val)* 100
print(acc_gbk_model )
train_df = pd.read_csv(".. /input/train.csv" )<init_hyperparams>
compare =pd.DataFrame({
'model':['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest',
'Decision Tree', 'Stochastic Gradient Descent', 'Gradient Boosting Classifier'],
'Score': [acc_svc, acc_knn_model, acc_log,
acc_randomforest, acc_decisiontree_model,
acc_sgd_model, acc_gbk_model]
} )
params = {
'boosting':'gbdt',
'bagging_freq':5,
'bagging_fraction':0.5,
'num_leaves':2,
'reg_lambda':100.0,
'learning_rate':0.01,
'max_bin':1023,
'seed':3366
}<train_on_grid>
ids = test['PassengerId']
predictions = gbk_model.predict(test.drop('PassengerId', axis=1))
output_file = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output_file.to_csv('submission.csv', index=False )