Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
import numpy as np
import pandas as pd
train['tagline_count'] = train['tagline'].apply(lambda x: 0 if pd.isnull(x)else len(x))
test['tagline_count'] = test['tagline'].apply(lambda x: 0 if pd.isnull(x)else len(x))<drop_column>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )
columns_to_keep.add('tagline_count' )<feature_engineering>
women = train_data[train_data['Sex'] == 'female']['Survived']
rate_women = sum(women)/len(women)
 print('% of women who survived:', rate_women )
train['title_count'] = train['title'].apply(lambda x: 0 if pd.isnull(x)else len(x))
test['title_count'] = test['title'].apply(lambda x: 0 if pd.isnull(x)else len(x))<feature_engineering>
men = train_data[train_data.Sex == 'male']['Survived']
rate_men = sum(men)/len(men)
 print('% of men who survived:', rate_men )
train['budget_popularity'] = train['budget']/train['popularity']
test['budget_popularity'] = test['budget']/test['popularity']
columns_to_keep.add('budget_popularity')
 train['budget_release_year'] = train['budget']/train['release_date_year']
test['budget_release_year'] = test['budget']/test['release_date_year']
columns_to_keep.add('budget_release_year')
 train['popularity_release_year'] = train['popularity']/train['release_date_year']
test['popularity_release_year'] = test['popularity']/test['release_date_year']
columns_to_keep.add('popularity_release_year')
 train['popularity_release_weekday'] = np.sqrt(train['popularity']*train['release_date_weekday'])
 test['popularity_release_weekday'] = np.sqrt(test['popularity']*test['release_date_weekday'])
 columns_to_keep.add('popularity_release_weekday')
 train['genres_number_release_year'] = train['genres_number']/train['release_date_year']
test['genres_number_release_year'] = test['genres_number']/test['release_date_year']
columns_to_keep.add('genres_number_release_year')
 train['runtime_release_year'] = np.sqrt(train['runtime']*train['release_date_year'])
 test['runtime_release_year'] = np.sqrt(test['runtime']*test['release_date_year'])
 columns_to_keep.add('runtime_release_year')
 train['budget_runtime'] = np.sqrt(train['budget']*train['runtime'])
 test['budget_runtime'] = np.sqrt(test['budget']*test['runtime'])
 columns_to_keep.add('budget_runtime')
 train['budget_tagline_count'] = np.sqrt(train['budget']*train['tagline_count'])
 test['budget_tagline_count'] = np.sqrt(test['budget']*test['tagline_count'])
 columns_to_keep.add('budget_tagline_count' )<define_variables>
train_data[['Sex', 'Survived']].groupby(['Sex'] ).mean()
numerical_features = ['Keywords_number','runtime','spoken_languages_number','production_countries_number',
'production_companies_number','popularity','genres_number','crew_number','cast_number','budget_log',
'budget_popularity','budget_release_year','popularity_release_year','popularity_release_weekday',
'genres_number_release_year','runtime_release_year','budget_runtime','budget_tagline_count']
date_features = ['release_date_day','release_date_month','release_date_weekday','release_date_weekofyear',
'release_date_year']
feature_text_length = ['title_count','tagline_count','original_title_length']
categorical_features = ['spoken_languages_all','production_companies_all','production_countries_all','original_language',
'got_collection','genres_all','crew_all','cast_all','belongs_to_three_lang','Keywords_all']
spoken_language_features = ['spoken_languages_','spoken_languages_Deutsch','spoken_languages_English','spoken_languages_Español',
'spoken_languages_Français','spoken_languages_Italiano', 'spoken_languages_Pусский',
'spoken_languages_हिन्दी','spoken_languages_日本語','spoken_languages_普通话']
production_countries_features = ['production_countries_Australia','production_countries_Canada','production_countries_France',
'production_countries_Germany','production_countries_India','production_countries_Italy',
'production_countries_Japan','production_countries_Russia','production_countries_United Kingdom',
'production_countries_United States of America' ]
production_companies_features = ['production_companies_Columbia Pictures','production_companies_Columbia Pictures Corporation',
'production_companies_Metro-Goldwyn-Mayer(MGM)', 'production_companies_New Line Cinema',
'production_companies_Paramount Pictures', 'production_companies_Touchstone Pictures',
'production_companies_Twentieth Century Fox Film Corporation','production_companies_Universal Pictures',
'production_companies_Walt Disney Pictures', 'production_companies_Warner Bros.']
genres_features = ['genres_Action', 'genres_Adventure','genres_Comedy', 'genres_Crime', 'genres_Drama', 'genres_Family',
'genres_Horror', 'genres_Romance', 'genres_Science Fiction', 'genres_Thriller']
crew_features = ['crew_Avy Kaufman', 'crew_Deborah Aquila', 'crew_Francine Maisler','crew_James Newton Howard',
'crew_Jerry Goldsmith', 'crew_Luc Besson', 'crew_Mary Vernieu', 'crew_Robert Rodriguez','crew_Steven Spielberg',
'crew_Tricia Wood']
cast_features = ['cast_Bruce McGill', 'cast_Bruce Willis','cast_J.K.Simmons','cast_John Turturro','cast_Liam Neeson',
'cast_Morgan Freeman','cast_Robert De Niro','cast_Samuel L.Jackson', 'cast_Susan Sarandon','cast_Willem Dafoe']
keywords_features = ['Keywords_aftercreditsstinger', 'Keywords_based on novel', 'Keywords_biography',
'Keywords_duringcreditsstinger', 'Keywords_independent film', 'Keywords_murder',
'Keywords_revenge', 'Keywords_sport', 'Keywords_violence', 'Keywords_woman director'] 
<concatenate>
train_data[['Pclass', 'Survived']].groupby(['Pclass'] ).mean()
numerical_data = numerical_features
numerical_data.append('revenue' )<choose_model_class>
train_data[['Age', 'Survived']].groupby(['Age'] ).mean()
def select_model(X_train, X_val, y_train, y_val):
best_models = {}
models = [
{
'name': 'LinearRegression',
'estimator': LinearRegression() ,
'hyperparameters': {},
},
{
'name': 'GradientBoostingRegressor',
'estimator': GradientBoostingRegressor() ,
'hyperparameters':{
'n_estimators': range(100, 200, 10),
'criterion': ['friedman_mse'],
'max_depth': [3, 5, 7, 9],
'max_features': ['log2', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [3, 5, 7]
}
},
{
'name': 'XGBoost',
'estimator': xgb.XGBRegressor() ,
'hyperparameters':{
'booster': ['gbtree', 'gblinear', 'dart'],
'max_depth': range(5, 50, 5),
'n_estimators': [200],
'nthread': [4],
'min_child_weight': range(1, 8, 2),
'learning_rate': [.05,.1,.15],
}
},
{
'name': 'Light GBM',
'estimator': lgb.LGBMRegressor() ,
'hyperparameters':{
'max_depth': range(20, 85, 15),
'learning_rate': [.01,.05,.1],
'num_leaves': [300, 600, 900, 1200],
'n_estimators': [200]
}
}
]
for model in tqdm(models):
print('
', '-'*25, '
', model['name'])
 start = time.perf_counter()
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5, scoring = "neg_mean_squared_error", verbose=False, n_jobs=-1)
 grid.fit(X_train, y_train)
 best_models[model['name']] = {'score': grid.best_score_, 'params': grid.best_params_}
mse_val = mean_squared_error(y_val, grid.predict(X_val))
mse_train = mean_squared_error(y_train, grid.predict(X_train))
print("RMSLE train:{}".format(np.sqrt(mse_train)))
 print("RMSLE validation:{}".format(np.sqrt(mse_val)))
 print("best_params_:{}".format(grid.best_params_))
run = time.perf_counter() - start
return best_models<split>
train_data[['Embarked', 'Survived']].groupby(['Embarked'] ).mean()
def get_best_parameters(train,features_list):
X_train = train[features_list]
y_train = np.log1p(train["revenue"] ).values
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=1)
 models = select_model(X_train, X_val, y_train, y_val)
 return models<train_model>
train_data[['Embarked', 'Pclass']].groupby(['Embarked'] ).mean()
class Models(object):
def __init__(self, x_train, x_validation, y_train, y_validation):
self.x_train = [x_train.iloc[i].tolist() for i in range(len(x_train)) ]
self.x_validation = [x_validation.iloc[i].tolist() for i in range(len(x_validation)) ]
self.y_train = y_train.tolist()
self.y_validation = y_validation.tolist()
@staticmethod
def print_info(cross_val_scores, mse_train,mse_val):
print("Cross Validation Scores: ", cross_val_scores)
 print("RMSLE train:{}".format(np.sqrt(mse_train)))
 print("RMSLE validation:{}".format(np.sqrt(mse_val)))
 def GBR(self, x_train, x_validation, y_train, y_validation):
gbr = GradientBoostingRegressor(n_estimators=120, learning_rate=0.08,max_features='sqrt',criterion='friedman_mse',
min_samples_leaf=1,min_samples_split=3, max_depth=7, random_state=seed)
 gbr.fit(self.x_train, self.y_train)
 kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
 cross_val_scores = cross_val_score(gbr, self.x_train, self.y_train, cv=kfold)
 mse_val = mean_squared_error(self.y_validation, gbr.predict(self.x_validation))
mse_train = mean_squared_error(self.y_train, gbr.predict(self.x_train))
print('
Gradient Boosting Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, gbr
def lgbm(self, x_train, x_validation, y_train, y_validation):
lgbm =lgb.LGBMRegressor(n_estimators=10000,objective="regression", metric="rmse",num_leaves=20,
min_child_samples=100,learning_rate=0.01, bagging_fraction=0.8,feature_fraction=0.8,
bagging_frequency=1,importance_type='gain', bagging_seed=seed,subsample=.9,
colsample_bytree=.9,use_best_model=True)
 lgbm.fit(x_train, y_train,eval_set=(x_validation, y_validation),verbose=False)
 kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
 cross_val_scores = cross_val_score(lgbm, self.x_train, self.y_train, cv=kfold)
 mse_val = mean_squared_error(self.y_validation, lgbm.predict(self.x_validation))
mse_train = mean_squared_error(self.y_train, lgbm.predict(self.x_train))
print('
LGBM Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, lgbm
def xgBoost(self, x_train, x_validation, y_train, y_validation):
params = {'objective': 'reg:linear','eta': 0.01,'max_depth': 6,'subsample': 0.6,'colsample_bytree': 0.7,
'eval_metric': 'rmse', 'seed': seed,'silent': True,}
record = dict()
xg = xgb.train(params, xgb.DMatrix(x_train, y_train), 100000, [(xgb.DMatrix(x_train, y_train), 'train'), 
(xgb.DMatrix(x_validation, y_validation), 'valid')]
, verbose_eval=False, early_stopping_rounds=500, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
cross_val_scores= 0
mse_val = mean_squared_error(self.y_validation, xg.predict(xgb.DMatrix(x_validation), ntree_limit=xg.best_ntree_limit))
mse_train = mean_squared_error(self.y_train, xg.predict(xgb.DMatrix(x_train), ntree_limit=xg.best_ntree_limit))
print('
XgBoost Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, xg<split>
women_count = 0
women_survived_count = 0
for idx, row in train_data.iterrows() :
if row['Sex'] == 'female':
women_count += 1
if row['Survived'] == 1:
women_survived_count += 1
women_survived_count / women_count
def evaluate_models(train, test,features_list):
X_train = train[features_list]
y_train = np.log1p(train["revenue"] ).values
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=1)
 methods = [x for x, y in Models.__dict__.items() if type(y)== FunctionType]
methods.remove('__init__')
 cross_scores_list, mse_list = [], []
models = {}
for model in methods:
reg = Models(X_train, X_val, y_train, y_val)
 cross_val_scores, mse, return_model = getattr(reg, model )(X_train, X_val, y_train, y_val)
 cross_scores_list.append(cross_val_scores)
 models[model] = return_model
mse_list.append(mse)
 return models<compute_train_metric>
predictions = []
for idx, row in test_data.iterrows() :
if row['Sex'] == 'female':
if row['Pclass'] == '1':
predictions.append(1)
 elif row['Pclass'] == '2':
if row['Embarked'] == 'C':
predictions.append(1)
 else:
predictions.append(0)
 else:
if row['Embarked'] == 'C' or row['Fare'] > 8:
predictions.append(1)
 else:
predictions.append(0)
 else:
if row['Age'] < 10 and row['Pclass'] != '3':
predictions.append(1)
 else:
predictions.append(0 )
corr_features_models = evaluate_models(train, test,corr_selected_features )<split>
test_data['Survived'] = predictions
<predict_on_test><EOS> 
test_data[['PassengerId', 'Survived']].to_csv('submission.csv', index=False )