Kaggle competition: Mercedes-Benz Greener Manufacturing
y_test = RFM.predict(dfm_test.drop(columns='PassengerId')).astype('int')
 results = pd.DataFrame(data={'PassengerId':dfm_test['PassengerId'].astype('int'), 'Survived':y_test})
 results.to_csv('Titanic PredictionETC.csv', index=False )<load_from_csv>
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error as MSE
from sklearn.metrics import r2_score
from sklearn.random_projection import GaussianRandomProjection
from sklearn.random_projection import SparseRandomProjection
from sklearn.decomposition import PCA
from sklearn.decomposition import FastICA
from sklearn.decomposition import TruncatedSVD
import lightgbm as lgb
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Lasso
%matplotlib inline
test_full = pd.read_csv(".. /input/titanic/test.csv",index_col="PassengerId")
 train_full = pd.read_csv(".. /input/titanic/train.csv",index_col = "PassengerId")
 target = train_full.Survived
train = train_full.drop(['Name','Survived'],axis=1)
 test = test_full.drop(['Name'],axis=1)
 sns.pairplot(train_full)
<count_unique_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
[(col,train[col].nunique())for col in train.select_dtypes('object')]<count_missing_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train.isnull().sum()<count_missing_values>
for f in object_list:
le = LabelEncoder()
le.fit(list(train[f].values)+ list(test[f].values))
train[f] = le.transform(list(train[f].values))
test[f] = le.transform(list(test[f].values))
Y = train['y']
train.drop(columns=['y'] , inplace = True , axis=1)
 combine = pd.concat([train , test])
 for f in object_list:
temp = pd.get_dummies(combine[f])
 combine = pd.concat([combine,temp] , axis =1)
 train=combine[:train.shape[0]]
test=combine[train.shape[0]:]
train.isnull().sum()<categorify>
def df_column_uniquify(df):
df_columns = df.columns
new_columns = []
for item in df_columns:
counter = 0
newitem = item
while newitem in new_columns:
counter += 1
newitem = "{}_{}".format(item, counter)
 new_columns.append(newitem)
 df.columns = new_columns
return df
train = df_column_uniquify(train)
 test = df_column_uniquify(test)
 train['y']=Y
imputer = SimpleImputer(missing_values=np.nan,strategy='most_frequent',add_indicator=True)
 train_preprocessed = pd.DataFrame(imputer.fit_transform(train),columns=[col for col in train] + ['Age_na','Cabin_na','Embarked_na'],index = train.index)
 test_preprocessed = pd.DataFrame(imputer.transform(test),columns=[col for col in test] + ['Age_na','Cabin_na','Embarked_na'],index = test.index)
 train_preprocessed<categorify>
original_col = list(test.drop(columns=object_list ).columns)
 display(train.head())
 display(test.head() )
LE = LabelEncoder()
for col in [col for col in train.select_dtypes('object')]:
train_preprocessed[col] = LE.fit_transform(train_preprocessed[col])
 test_preprocessed[col] = LE.fit_transform(test_preprocessed[col])
 train_preprocessed = train_preprocessed.astype('float64')
 test_preprocessed = test_preprocessed.astype('float64')
 train_preprocessed<split>
def get_additional_features(train , test , ID = False):
col = list(test.columns)
 if ID!= True:
col.remove('ID')
 n_comp = 12
tsvd = TruncatedSVD(n_components = n_comp , random_state = 98)
 tsvd_result_train = tsvd.fit_transform(train[col])
 tsvd_result_test = tsvd.transform(test[col])
 pca = PCA(n_components = n_comp , random_state = 98)
 pca_result_train = pca.fit_transform(train[col])
 pca_result_test = pca.transform(test[col])
 ica = FastICA(n_components =n_comp , random_state = 98)
 ica_result_train = ica.fit_transform(train[col])
 ica_result_test = ica.transform(test[col])
 grp = GaussianRandomProjection(n_components = n_comp , random_state = 98)
 grp_result_train = grp.fit_transform(train[col])
 grp_result_test = grp.transform(test[col])
 srp = SparseRandomProjection(n_components = n_comp , random_state = 98 , dense_output =True)
 srp_result_train = srp.fit_transform(train[col])
 srp_result_test = srp.transform(test[col])
 for i in range(1,n_comp+1):
train['tsvd_' + str(i)] = tsvd_result_train[:, i - 1]
test['tsvd_' + str(i)] = tsvd_result_test[:, i - 1]
train['pca_' + str(i)] = pca_result_train[:, i - 1]
test['pca_' + str(i)] = pca_result_test[:, i - 1]
train['ica_' + str(i)] = ica_result_train[:, i - 1]
test['ica_' + str(i)] = ica_result_test[:, i - 1]
train['grp_' + str(i)] = grp_result_train[:, i - 1]
test['grp_' + str(i)] = grp_result_test[:, i - 1]
train['srp_' + str(i)] = srp_result_train[:, i - 1]
test['srp_' + str(i)] = srp_result_test[:, i - 1]
return train ,test
X_train,X_test,y_train,y_test = train_test_split(train_preprocessed,target,test_size=0.2)
 display(X_train )<train_model>
def get_lgb_data(train , test , col , label , params ,rounds):
i=0
RMSE = []
R2_Score = []
ID = []
kf = KFold(n_splits = 5 , shuffle = False)
 train = train.reset_index(drop = True)
 for train_index , test_index in kf.split(train):
train_x , test_x = train.iloc[train_index, :] ,test.iloc[test_index ,:]
train_y , test_y = label.iloc[train_index] , label.iloc[test_index]
train_lgb = lgb.Dataset(train_x[col] , train_y)
 model = lgb.train(params , train_lgb , num_boost_round = rounds)
 pred = model.predict(test_x[col])
 test_x['label'] = list(test_y)
 test_x['predicted'] = pred
r2 = r2_score(test_y , pred)
 rmse = MSE(test_y ,pred)**0.5
print('R2 Scored of Fold '+str(i+1)+' is '+str(r2))
R2_Score.append(r2)
 RMSE.append(rmse)
 print('RMSE of Fold '+str(i+1)+' is '+str(rmse))
ID.append(test_x['ID'])
 if i==0:
Final = test_x
else:
Final = Final.append(test_x,ignore_index=True)
 i+=1
lgb_train = lgb.Dataset(train[col], label)
 model = lgb.train(params , lgb_train , num_boost_round = rounds)
 pred = model.predict(test[col])
 lgb.plot_importance(model, max_num_features = 20)
 Final_pred = pd.DataFrame({'ID': test['ID'] ,'y':pred})
 print('Out of Bag R2 Score')
 print(np.mean(r2))
print('Out of Bag RMSE')
 print(np.mean(RMSE))
return Final , Final_pred
model = XGBClassifier(n_estimators=90,n_jobs=5,random_state=0)
 model.fit(X_train,y_train)
 preds = model.predict(X_test)
 'Accuracy Score is %f' % accuracy_score(y_pred=preds,y_true=y_test )<compute_test_metric>
def get_sklearn_data(train , test , model , label ,col):
ID = []
RMSE = []
R2_Score = []
i=0
train =train.reset_index(drop = True)
 kf = KFold(n_splits = 5 , shuffle = False)
 for train_index , test_index in kf.split(train):
train_x , test_x = train.iloc[train_index,:] , train.iloc[test_index,:]
train_y , test_y = label.iloc[train_index] , label.iloc[test_index]
model.fit(train_x[col],train_y)
 pred = model.predict(test_x[col])
 test_x['label'] = list(test_y)
 test_x['predicted'] = pred
r2 = r2_score(test_y , pred)
 rmse = MSE(test_y , pred)**0.5
print('R2 Scored of Fold '+str(i+1)+' is '+str(r2))
R2_Score.append(r2)
 RMSE.append(rmse)
 print('RMSE of Fold '+str(i+1)+' is '+str(rmse))
ID.append(test_x['ID'])
 if i==0:
Final = test_x
else:
Final = Final.append(test_x,ignore_index=True)
 i+=1
model.fit(train[col] , label)
 Final_pred = model.predict(test[col])
 Final_pred = pd.DataFrame({'ID':test['ID'] , 'y':Final_pred})
 print('In of Bag R2 Score')
 print(r2_score(label , model.predict(train[col])))
 print('Out of Bag R2 Score')
 print(np.mean(R2_Score))
print('In of Bag MSE Score')
 print(MSE(label , model.predict(train[col])))
 print('Out of Bag MSE Score')
 print(np.mean(RMSE))
return Final , Final_pred , model
mean_absolute_error(preds,y_test )<compute_test_metric>
def magic_function(train ,test , columns):
for col in columns:
magic = train[['ID' , col , 'y']]
magic = magic.groupby([col])['y'].mean()
col_name = str(col)+'_mean'
magic = pd.DataFrame({col: magic.index , col_name:list(magic)})
 magic_mean = magic[col_name].mean()
train = train.merge(magic , how = 'left' , on = col)
 test = test.merge(magic , how = 'left' , on = col)
 test[col_name] = test[col_name].fillna(magic_mean)
 return train , test
f1_score(preds,y_test )<save_to_csv>
train, test = magic_function(train , test , object_list)
 train.sample(5)
 train_new , test_new = get_additional_features(train.drop(columns = object_list),test.drop(columns = object_list))
train_new = train_new.sample(frac = 1 , random_state = 98)
 train_new.sample(10)
submission_preds = model.predict(test_preprocessed)
 output = pd.read_csv(".. /input/titanic/gender_submission.csv")
 output.Survived = submission_preds
output.to_csv('submission.csv',index=False)
 output.head()<set_options>
train_new.drop(columns=columns_drop , inplace = True)
 test_new.drop(columns = columns_drop , inplace = True)
 col = list(test_new.columns )
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
%%time
lasso1 = Lasso(alpha = 5 , random_state = 98)
 lasso_train , lasso_test , model = get_sklearn_data(train_new , test_new , lasso1 , train_new['y'] , col)
test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv" )<count_missing_values>
%%time
rfr = RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
 rfr_train , rfr_test , model = get_sklearn_data(train_new , test_new , rfr , train_new['y'] , col)
print('Train')
 print(train.isnull().sum())
 print('==========================')
 print('Test')
 print(test.isnull().sum() )<categorify>
%%time
params = {
'objective': 'regression',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.0045 ,
'verbose': 0,
'num_iterations': 500,
'bagging_fraction': 0.95,
'bagging_freq': 1,
'bagging_seed': 42,
'feature_fraction': 0.95,
'feature_fraction_seed': 42,
'max_bin': 100,
'max_depth': 3,
'num_rounds': 800
}
lgb_train, lgb_test = get_lgb_data(train_new, test_new , col , train_new['y'] , params , 800 )
train = train.fillna({'Age': -0.1})
 test = test.fillna({'Age': -0.1})
 train['Sex'] = LabelEncoder().fit_transform(train['Sex'])
 test['Sex'] = LabelEncoder().fit_transform(test['Sex'])
 train.loc[~train.Cabin.isnull() , 'Cabin'] = 1
train.loc[train.Cabin.isnull() , 'Cabin'] = 0
test.loc[~test.Cabin.isnull() , 'Cabin'] = 1
test.loc[test.Cabin.isnull() , 'Cabin'] = 0
common_embarked = train.groupby(['Embarked'])['Embarked'].value_counts().idxmax() [0]
train = train.fillna({'Embarked': common_embarked})
 test = test.fillna({'Embarked': common_embarked})
 test.loc[test.Fare.isnull() , 'Fare'] = 0<feature_engineering>
train_new , test_new = get_additional_features(train.drop(columns = object_list),test.drop(columns = object_list), ID = True)
 train_new = train_new.sample(frac = 1 , random_state = 98)
 train_new.drop(columns=columns_drop , inplace = True)
 test_new.drop(columns = columns_drop , inplace = True)
 col = list(test_new.columns)
 y_mean = np.mean(train_new['y'])
 xgb_params = {
'n_trees': 520,
'eta': 0.0045,
'max_depth': 4,
'subsample': 0.93,
'eval_metric': 'rmse',
'base_score': y_mean,
'silent': True,
'seed': 42,
}
dtrain = xgb.DMatrix(train_new[col], train_new.y)
 dtrain_test = xgb.DMatrix(train_new[col])
 dtest = xgb.DMatrix(test_new[col])
 num_boost_rounds = 1250
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_rounds)
 y_pred = model.predict(dtest )
train['Title'] = train.Name.str.split(',', n=1, expand=True)[1].str.split('.',n=1, expand=True)[0]
train['Title'] = train.Title.str.strip()
test['Title'] = test.Name.str.split(',', n=1, expand=True)[1].str.split('.', n=1, expand=True)[0]
test['Title'] = test.Title.str.strip()
train.head()<feature_engineering>
stack_train = gb1_train[['label' , 'predicted']]
stack_train.columns = [['label','gbrp']]
stack_train['lgb'] = lgb_train['predicted']
stack_train['lasso'] = lasso_train['predicted']
stack_test = gb1_test[['ID' , 'y']]
stack_test.columns = [['ID' , 'gbrp']]
stack_test['lgb'] = lgb_test['y']
stack_test['lasso'] = lasso_test['y']
stack_test = stack_test.drop(['ID'] ,axis = 1)
 col = list(stack_test.columns)
 params = {
'eta': 0.005,
'max_depth': 2,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(stack_train[col], stack_train['label'])
 dtest = xgb.DMatrix(stack_test[col])
 xgb_cvalid = xgb.cv(params, dtrain, num_boost_round=2000, early_stopping_rounds=20, verbose_eval=50, show_stdv=True,seed=42)
 xgb_cvalid[['train-rmse-mean', 'test-rmse-mean']].plot()
print('Performance does not improve from '+str(len(xgb_cvalid)) +' rounds')
 model = xgb.train(params,dtrain,num_boost_round =900)
 pred_1 = model.predict(dtest)
 xgb.plot_importance(model, max_num_features = 3)
 plt.show()
<feature_engineering><EOS> 
Average = 0.70*y_pred + 0.30*pred_1
sub = pd.DataFrame({'ID':test['ID'],'y':Average})
 sub1 = pd.DataFrame({'ID':test['ID'],'y':y_pred})
 sub2 = pd.DataFrame({'ID':test['ID'],'y':pred_1} )