Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
py.init_notebook_mode(connected=True)
 %matplotlib inline
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for col in train_texts.columns:
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=10 
)
 vectorizer.fit(list(train_texts[col].fillna('')) + list(test_texts[col].fillna('')))
 train_col_text = vectorizer.transform(train_texts[col].fillna(''))
test_col_text = vectorizer.transform(test_texts[col].fillna(''))
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_text, prediction_text = train_model(train_col_text, test_col_text, y, params=None, model_type='sklearn', model=model)
 X[col + '_oof'] = oof_text
X_test[col + '_oof'] = prediction_text<feature_engineering>
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 PassengerId=test['PassengerId']
train.head()
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df<feature_engineering>
full_data=[train,test]
train['name_length']=train['Name'].apply(len)
 test['name_length']=test['Name'].apply(len)
 train['has_cabin']=train["Cabin"].apply(lambda x:0 if type(x)==float else 1)
 test['has_cabin']=test["Cabin"].apply(lambda x:0 if type(x)==float else 1)
 for dataline in full_data:
dataline["familysize"]=dataline['SibSp']+dataline['Parch']+1
for dataline in full_data:
dataline["isalone"]=0
dataline.loc[dataline['familysize']==1,'isalone']=1
dataline['Embarked']= dataline['Embarked'].fillna('S')
 dataline['Fare']=dataline['Fare'].fillna(train['Fare'].median())
 train['categoricalfare']=pd.cut(train['Fare'],4)
 for dataset in full_data:
age_avg=dataset['Age'].mean()
age_std=dataset['Age'].std()
age_null_count=dataset['Age'].isnull().sum()
age_null_random_list=np.random.randint(age_avg-age_std,age_avg+age_std,size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])]=age_null_random_list
dataset['Age']=dataset['Age'].astype(int)
 train['categoricalage']=pd.cut(train['Age'],5)
 def get_title(name):
title_search=re.search('([A-Za-z]+)\.',name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title']=dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train.head()
X = new_features(X)
 X_test = new_features(X_test )<train_model>
for dataset in full_data:
dataset['Sex']=dataset['Sex'].map({'female':0,'male':1} ).astype(int)
 title_mapping={"Mr":1,"Miss":2,"Mrs":3,"Master":4,"Rare":5}
dataset['Title']=dataset['Title'].map(title_mapping)
 dataset['Title']=dataset['Title'].fillna(0)
 dataset['Embarked']=dataset['Embarked'].map({'S':0,'C':1,'Q':2})
 dataset.loc[dataset['Fare']<=7.91,'Fare']=0
dataset.loc[(dataset['Fare']>7.91)&(dataset['Fare']<=14.454),'Fare']=1
dataset.loc[(dataset['Fare']>14.454)&(dataset['Fare']<=31),'Fare']=2
dataset.loc[dataset['Fare']>31,'Fare']=3
dataset['Fare']=dataset['Fare'].astype(int)
 dataset.loc[dataset['Age']<=16,'Age']=0
dataset.loc[(dataset['Age']>16)&(dataset['Age']<=32),'Age']=1
dataset.loc[(dataset['Age']>32)&(dataset['Age']<=48),'Age']=2
dataset.loc[(dataset['Age']>48)&(dataset['Age']<=64),'Age']=3
dataset.loc[dataset['Age']>64,'Age']=4
train.head()
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )<split>
drop_elements=['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train=train.drop(drop_elements,axis=1)
 train=train.drop(['categoricalage', 'categoricalfare'],axis=1)
 test=test.drop(drop_elements,axis=1)
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)
 params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 7,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )<feature_engineering>
ntrain=train.shape[0]
ntest=test.shape[0]
SEED=0
nfolds=5
kf=KFold(n_splits=nfolds,random_state=SEED)
 class SKlearnHelper(object):
def __init__(self,clf,seed=0,params=None):
self.clf=clf(**params)
 def train(self,x_train,y_train):
self.clf.fit(x_train,y_train)
 def predict(self,x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
def top_cols_interaction(df):
df['budget_to_year'] = df['budget'] / df['release_date_year']
df['budget_to_mean_year_to_year'] = df['budget_to_mean_year'] / df['release_date_year']
df['popularity_to_mean_year_to_log_budget'] = df['popularity_to_mean_year'] / df['log_budget']
df['year_to_log_budget'] = df['release_date_year'] / df['log_budget']
df['budget_to_runtime_to_year'] = df['budget_to_runtime'] / df['release_date_year']
df['genders_1_cast_to_log_budget'] = df['genders_1_cast'] / df['log_budget']
df['all_genres_to_popularity_to_mean_year'] = df['all_genres'] / df['popularity_to_mean_year']
df['genders_2_crew_to_budget_to_mean_year'] = df['genders_2_crew'] / df['budget_to_mean_year']
df['overview_oof_to_genders_2_crew'] = df['overview_oof'] / df['genders_2_crew']
return df<concatenate>
def get_oof(clf,x_train,y_train,x_test):
oof_train=np.zeros(( ntrain,))
oof_test=np.zeros(( ntest,))
oof_test_skf=np.empty(( nfolds,ntest))
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
x_tr=x_train[train_index]
y_tr=y_train[train_index]
x_te=x_train[test_index]
clf.train(x_tr,y_tr)
 oof_train[test_index]=clf.predict(x_te)
 oof_test_skf[i,:]=clf.predict(x_test)
 oof_test[:]=oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1,1),oof_test.reshape(-1,1)
X = top_cols_interaction(X)
 X_test = top_cols_interaction(X_test )<categorify>
rf_params={
'n_jobs':-1,
'n_estimators':500,
'warm_start':True,
'max_depth':6,
'min_samples_leaf':2,
'max_features':'sqrt',
'verbose':0
}
et_params={
'n_jobs':-1,
'n_estimators':500,
'max_depth':8,
'min_samples_leaf':2,
'verbose':0
}
ada_params={
'n_estimators':500,
'learning_rate':0.75
}
gb_params={
'n_estimators':500,
'max_depth':8,
'min_samples_leaf':2,
'verbose':0
}
svc_params={
'kernel':'linear',
'C':0.025
}
knn_params={
'n_neighbors':3
}
dst_params={
'max_depth':8
}
X = X.replace([np.inf, -np.inf], 0 ).fillna(0)
 X_test = X_test.replace([np.inf, -np.inf], 0 ).fillna(0 )<load_from_csv>
rf=SKlearnHelper(clf=RandomForestClassifier,seed=SEED,params=rf_params)
 et=SKlearnHelper(clf=ExtraTreesClassifier,seed=SEED,params=et_params)
 ada=SKlearnHelper(clf=AdaBoostClassifier,seed=SEED,params=ada_params)
 gb=SKlearnHelper(clf=GradientBoostingClassifier,seed=SEED,params=gb_params)
 svc=SKlearnHelper(clf=SVC,seed=SEED,params=svc_params)
 knn=SKlearnHelper(clf=KNeighborsClassifier,seed=SEED,params=knn_params)
 dst=SKlearnHelper(clf=DecisionTreeClassifier,seed=SEED,params=dst_params )
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 X['imdb_id'] = train['imdb_id']
X_test['imdb_id'] = test['imdb_id']
del train, test
X = pd.merge(X, trainAdditionalFeatures, how='left', on=['imdb_id'])
 X_test = pd.merge(X_test, testAdditionalFeatures, how='left', on=['imdb_id'])
 X = X.drop(['imdb_id'], axis=1)
 X_test = X_test.drop(['imdb_id'], axis=1 )<train_model>
y_train=train['Survived'].ravel()
train=train.drop(['Survived'],axis=1)
 x_train=train.values
x_test=test.values
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 9,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )<train_model>
et_oof_train,et_oof_test=get_oof(et,x_train,y_train,x_test)
 rf_oof_train,rf_oof_test=get_oof(rf,x_train,y_train,x_test)
 ada_oof_train,ada_oof_test=get_oof(ada,x_train,y_train,x_test)
 gb_oof_train,gb_oof_test=get_oof(gb,x_train,y_train,x_test)
 svc_oof_train,svc_oof_test=get_oof(svc,x_train,y_train,x_test)
 knn_oof_train,knn_oof_test=get_oof(knn,x_train,y_train,x_test)
 dst_oof_train,det_oof_test=get_oof(dst,x_train,y_train,x_test)
 print("training is complete" )
xgb_params = {'eta': 0.01,
'objective': 'reg:linear',
'max_depth': 5,
'subsample': 0.8,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': 11,
'silent': True}
oof_xgb, prediction_xgb = train_model(X, X_test, y, params=xgb_params, model_type='xgb', plot_feature_importance=False )<train_model>
base_prediction_train=pd.DataFrame({'randomforest':rf_oof_train.ravel() ,
'extratrees':et_oof_train.ravel() ,
'adaboost':ada_oof_train.ravel() ,
'gradientboost':gb_oof_train.ravel() ,
})
 base_prediction_train.head()
cat_params = {'learning_rate': 0.02,
'depth': 5,
'l2_leaf_reg': 10,
'bootstrap_type': 'Bernoulli',
'od_type': 'Iter',
'od_wait': 50,
'random_seed': 11,
'allow_writing_files': False}
oof_cat, prediction_cat = train_model(X, X_test, y, params=cat_params, model_type='cat' )<train_model>
x_train=np.concatenate(( et_oof_train,rf_oof_train,gb_oof_train,ada_oof_train,svc_oof_train,knn_oof_train,dst_oof_train),axis=1)
 x_test=np.concatenate(( et_oof_test,rf_oof_test,gb_oof_test,ada_oof_test,svc_oof_test,knn_oof_test,det_oof_test),axis=1)
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 3,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_1, prediction_lgb_1 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )<train_model>
gbm=xgb.XGBClassifier(
 n_estimators=2000,
max_depth=4,
min_child_weight=2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective='binary:logistic',
nthread=-1,
scale_pos_weight=1 ).fit(x_train,y_train)
 predictions=gbm.predict(x_test)
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 7,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.7,
"bagging_freq": 5,
"bagging_fraction": 0.7,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_2, prediction_lgb_2 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )<create_dataframe>
stackingsubmission=pd.DataFrame({'PassengerId':PassengerId,'Survived':predictions})
 stackingsubmission.to_csv("stackingsubmission.csv",index=False )