Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
simplefilter(action='ignore', category=FutureWarning)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118<feature_engineering>
def label_encode(df, column_name):
ordered_column = np.sort(df[column_name].unique())
 df[column_name] = df[column_name].map(
 dict(zip(np.sort(df[column_name].unique()),[x for x in range(len(df[column_name].unique())) ])) 
)
 return df
def compare(df,column_name, with_table=False, with_graph=True, compare_to='Survived'):
if with_table:
print(df[df[compare_to] < 3].groupby([compare_to,column_name] ).size().sort_index())
 if with_graph:
g = sns.FacetGrid(df, col=compare_to ).map(sns.distplot, column_name)
 def show_correlation(df, column_name='Survived'):
return df.corr() [column_name].apply(abs ).sort_values(na_position='first' ).reset_index()
def get_IQR(df, column_name):
Q3 = df[column_name].quantile(0.75)
 Q1 = df[column_name].quantile(0.25)
 IQR = Q3 - Q1
return Q1, Q3, IQR
def detect_outliers(df, n, features):
outlier_indices = []
for col in features:
Q1, Q3, IQR = get_IQR(df, col)
 outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<save_to_csv>
train_df = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_df['Survived'] = 3
train_df = train_df[train_df.columns]
test_df = test_df[train_df.columns]
all_set = pd.concat([train_df, test_df], ignore_index=True )
if not debug:
sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f' )<feature_engineering>
def get_title(name):
return name.split(',')[1].split('.')[0].strip()
all_set['Title'] = all_set['Name'].apply(get_title )
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates() , on='building_id', how='left' )<compute_test_metric>
all_set['Surname'] = all_set['Name'].apply(lambda x: x.split(',')[0].strip() )
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))<feature_engineering>
all_set['FamilySurvival'] = 0.5
for surname in all_set['Surname'].unique() :
df = all_set[all_set['Surname'] == surname]
if df.shape[0] > 1:
smin = df['Survived'].min()
smax = df['Survived'].max()
for idx, row in df.iterrows() :
passengerid = row['PassengerId']
if smax == 1.0:
all_set.loc[all_set['PassengerId'] == passengerid, 'FamilySurvival'] = 1.0
elif smin == 0.0:
all_set.loc[all_set['PassengerId'] == passengerid, 'FamilySurvival'] = 0.0
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
all_set['Embarked'].value_counts()
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<compute_test_metric>
all_set['Embarked'] = all_set['Embarked'].fillna('S' )
print('LV score=', leak_score )<define_variables>
all_set[all_set['Fare'].isna() ]
DATA_PATH = ".. /input/ashrae-energy-prediction/"<load_from_csv>
all_set['Fare'].fillna(all_set[all_set['Embarked'] == 'S']['Fare'].mean() , inplace=True )
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )<data_type_conversions>
all_set['Deck'] = all_set['Cabin'].apply(lambda x: x[0] if type(x)== str else '' )
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df<create_dataframe>
all_set[all_set['Deck'] == 'G']
weather_df = fill_weather_dataset(weather_df )<categorify>
deck_fare = {}
for deck in all_set['Deck'].unique() :
if len(deck)== 1 and deck in 'ABCDEF':
deck_fare[deck] = all_set[ 
(all_set['Cabin'].apply(lambda x: True if type(x)== str else False)) & 
(all_set['Deck'] == deck)
 ]['Fare'].mean()
deck_fare
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )<merge>
def find_deck(fare):
dist = 1000
res = 'F'
for key in deck_fare.keys() :
new_dist = np.abs(fare - deck_fare[key])
 if new_dist < dist:
dist = new_dist
res = key
return res
all_set.loc[all_set['Cabin'].isna() , 'Deck'] = all_set['Fare'].apply(find_deck )
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()<feature_engineering>
all_set['Family'] = 1 + all_set['SibSp'] + all_set['Parch']
all_set['Alone'] = all_set['Family'].apply(lambda x: 1 if x == 1 else 0 )
train_df = features_engineering(train_df )<prepare_x_and_y>
age_by_title = {}
for title in all_set['Title'].unique() :
age_by_title[title] = all_set[ 
(all_set['Age'].apply(lambda x: True if type(x)== float else False)) & 
(all_set['Title'] == title)
 ]['Age'].mean()
age_by_title
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df
gc.collect()<split>
all_set.loc[all_set['Age'].isna() , 'Age'] = all_set['Title'].apply(lambda x: age_by_title[x] )
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend"]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()<set_options>
all_set['AgeBin'] = pd.qcut(all_set['Age'], 5)
 all_set['AgeCode'] = all_set['AgeBin']
all_set = label_encode(all_set, 'AgeCode' )
del features, target
gc.collect()<load_from_csv>
all_set['AgeDist'] = 1/ np.exp(np.abs(all_set['Age'] - all_set[all_set['Survived'] == 1]['Age'].median()))
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )<merge>
all_set = label_encode(all_set, 'Sex')
 compare(all_set, 'Sex' )
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()<load_from_csv>
all_set = label_encode(all_set, 'Embarked')
 compare(all_set, 'Embarked' )
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
<merge>
all_set = label_encode(all_set, 'Deck')
 compare(all_set, 'Deck' )
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()<feature_engineering>
all_set['Title'].value_counts()
test_df = features_engineering(test_df )<predict_on_test>
dict(zip(all_set['Title'].unique() ,['Rare' for x in range(len(all_set['Title'].unique())) ]))
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()<set_options>
all_set['Title'] = all_set['Title'].map({
'Mr': 'Mr', 'Mrs': 'Mrs', 'Miss': 'Miss', 'Master': 'Master',
'Don': 'Rare', 'Rev': 'Rare', 'Dr': 'Rare', 'Mme': 'Rare',
'Ms': 'Rare', 'Major': 'Rare', 'Lady': 'Rare', 'Sir': 'Rare',
'Mlle': 'Rare', 'Col': 'Rare', 'Capt': 'Rare', 'the Countess': 'Rare',
'Jonkheer': 'Rare', 'Dona': 'Rare'} )
del test_df, models
gc.collect()<save_to_csv>
all_set = label_encode(all_set, 'Title')
 compare(all_set, 'Title' )
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)
<import_modules>
all_set['FarePerFamilyMember'] = all_set['Fare'] / all_set['Family']
all_set['FareBin'] = pd.qcut(all_set['Fare'], 5)
 all_set['FareCode'] = all_set['FareBin']
all_set = label_encode(all_set, 'FareCode')
 all_set['Fare'] = all_set['Fare'].apply(lambda x: np.log(x)if x > 0 else np.log(3.0))
import plotly.express as px
import plotly.graph_objects as go
import catboost as cgb
from catboost import CatBoostRegressor,Pool
import shap
from bayes_opt import BayesianOptimization
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error<load_from_csv>
all_tmp = all_set.drop(columns=['PassengerId', 'Name', 'Ticket', 'Cabin', 'AgeBin', 'AgeCode', 'AgeDist', 'FareBin', 'FareCode'] )
X=pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 X.head()<load_from_csv>
all_dum = pd.get_dummies(all_tmp, drop_first=True)
 print(all_dum.shape)
 train_set = all_dum[all_dum['Survived'] < 3].copy()
test_set = all_dum[all_dum['Survived'] == 3].copy()
X_test=pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 X_test.head()<sort_values>
X = train_set.copy()
try:
X.drop(columns=['Survived'], inplace=True)
 except:
pass
y = train_set['Survived'].copy()
scaler = StandardScaler().fit(X)
 scl_X = scaler.transform(X)
 tX = test_set.copy()
try:
tX.drop(columns=['Survived'], inplace=True)
 except:
pass
scl_tX = scaler.transform(tX )
corr=X.corr()
corr_t=corr['target'].sort_values(ascending=False)
 corr_t<drop_column>
train_X, val_X, train_y, val_y = train_test_split(scl_X, y, test_size=0.33, random_state=42, stratify=y )
X=X.drop('id',axis=1 )<split>
rfc = RandomForestClassifier(n_estimators=100)
 rfc.fit(train_X, train_y)
 score = rfc.score(val_X, val_y)
 print('RandomForestClassifier =', score)
 pred_y = rfc.predict(val_X)
 target_names = ['DEAD', 'SURVIVED']
print(classification_report(val_y, pred_y,target_names=target_names))
y=X.target
X=X.drop('target',axis=1)
 X_train,X_val,y_train,y_val=train_test_split(X,y,test_size=0.2,random_state=1 )<choose_model_class>
best_model = 0
best_score = score
def objective(max_depth,random_strength,learning_rate,iterations,bagging_temperature):
params={'eval_metric':'RMSE',
'verbose':False,
'loss_function':'RMSE',
'num_leaves':31,
'bootstrap_type':'Bayesian'
}
params['max_depth']=int(round(max_depth))
params['random_strength']=random_strength
params['learning_rate']=learning_rate
params['iterations']=int(round(iterations))
params['bagging_temperature']=bagging_temperature
cv_dataset=cgb.Pool(data=X,label=y)
 scores=cgb.cv(cv_dataset,params,fold_count=3,)
 return np.min(scores['test-RMSE-mean'])
 pds={
'max_depth':(3,10),
'random_strength':(1,10),
'learning_rate':(0.01,0.3),
'iterations':(100,1000),
'bagging_temperature':(3,10)
 }
optimizer=BayesianOptimization(objective,pds,random_state=2100)
 optimizer.maximize(init_points=3,n_iter=7 )<train_model>
clfs = []
clfs.append(( 'ada', AdaBoostClassifier))
clfs.append(( 'bag', BaggingClassifier))
clfs.append(( 'rnd', RandomForestClassifier))
clfs.append(( 'knn', KNeighborsClassifier))
clfs.append(( 'mlp', MLPClassifier))
clfs.append(( 'ext', ExtraTreesClassifier))
clfs.append(( 'log', LogisticRegression))
clfs.append(( 'gbm', GradientBoostingClassifier))
params = []
params.append({'n_estimators': np.arange(10,500,10), 'learning_rate':[float(x/100.) for x in np.arange(1,10)]})
 params.append({'n_estimators': np.arange(10,500,10)})
 params.append({'n_estimators': np.arange(10,500,10)})
 params.append({'n_neighbors': np.arange(3,15)})
 params.append({'hidden_layer_sizes': [(100,),(200,),(300,),(400,),(500,)]})
 params.append({'n_estimators': np.arange(10,200,10)})
 params.append({'max_iter': np.arange(10,500,10)})
 params.append({'n_estimators': np.arange(10,500,10), 'learning_rate':[float(x/100.) for x in np.arange(1,10)], 'max_depth':np.arange(3,10)} )
ct=CatBoostRegressor(iterations=786
,learning_rate=0.08262,
max_depth=6,
num_leaves=31,
random_strength=1.427,
use_best_model=True,
eval_metric='RMSE',
verbose=False,
bagging_temperature=4.901)
 ct.fit(X_train,y_train,
verbose_eval=100,
early_stopping_rounds=30,
eval_set=(X_val,y_val),
plot=True);<predict_on_test>
best_estimators = []
best_params = []
estimators_weights = []
k = len(params)
 for idx in range(len(clfs)) :
gs = RandomizedSearchCV(clfs[idx][1]() , params[idx], cv=5)
 gs.fit(train_X, train_y)
 estimators_weights.append(gs.score(val_X, val_y))
best_estimators.append(gs.best_estimator_)
 best_params.append(gs.best_params_)
 print(k, clfs[idx][0], gs.best_params_)
 k -= 1
pred=ct.predict(X_val)
 print(np.sqrt(mean_squared_error(y_val,pred)) )<save_to_csv>
estimator_list = list(zip([name for(name, model)in clfs], best_estimators))
prediction_file=ct.predict(X_test.drop('id',axis=1))
submission=pd.DataFrame({
"id":X_test['id'],
"target":prediction_file
})
 submission.to_csv('my_submission.csv',index=False )<import_modules>
vclf = VotingClassifier(estimator_list, voting='hard', weights=estimators_weights, n_jobs=-1)
 vclf.fit(train_X, train_y)
 score = vclf.score(val_X, val_y)
 print('VotingClassifier =', score)
 if score > best_score:
best_model = 1
best_score = score
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from tqdm import tqdm<load_from_csv>
pred_y = vclf.predict(val_X)
 target_names = ['DEAD', 'SURVIVED']
print(classification_report(val_y, pred_y,target_names=target_names))
train_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
X_train = train_data[features]
y_train = train_data["target"]
final_X_test = test_data[features]
X_train,X_test,y_train,y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=0)
 regressor = xgb.XGBRegressor(
 colsample_bytree=0.5,
alpha=0.01563,
learning_rate=0.01,
max_depth=15,
min_child_weight=257,
n_estimators=4000,
reg_lambda=0.003,
subsample=0.7,
random_state=2020,
metric_period=100,
silent=1)
 regressor.fit(X_train, y_train, early_stopping_rounds=6, eval_set=[(X_test, y_test)], verbose=1 )<predict_on_test>
fit_X = np.zeros(( train_y.shape[0], len(best_estimators)))
 fit_X = pd.DataFrame(fit_X)
 pred_X = np.zeros(( val_y.shape[0], len(best_estimators)))
 pred_X = pd.DataFrame(pred_X)
 test_X = np.zeros(( scl_tX.shape[0], len(best_estimators)))
 test_X = pd.DataFrame(test_X)
 print("Fitting models.")
 cols = list()
for i,(name, m)in enumerate(estimator_list):
print("%s..." % name, end=" ", flush=False)
 fit_X.iloc[:, i] = m.predict_proba(train_X)[:, 1]
pred_X.iloc[:, i] = m.predict_proba(val_X)[:, 1]
test_X.iloc[:, i] = m.predict_proba(scl_tX)[:, 1]
cols.append(name)
 print("done")
 fit_X.columns = cols
pred_X.columns = cols
test_X.columns = cols
predictions = regressor.predict(final_X_test )<prepare_x_and_y>
meta_estimator = GradientBoostingClassifier()
meta_params = {'n_estimators': np.arange(10,500,10), 'learning_rate':[float(x/100.) for x in np.arange(1,10)], 'max_depth':np.arange(3,10)}
meta_estimator = RandomizedSearchCV(GradientBoostingClassifier() , meta_params, cv=5)
 meta_estimator.fit(fit_X, train_y)
 score = meta_estimator.score(pred_X, val_y)
 print('MetaEstimator =', score)
 if score > best_score:
best_model = 2
best_score = score
X = train_data.drop(['id','target'], axis=1)
 Xtest = test_data.drop(['id'], axis=1)
 y = train_data['target']
train = int(len(X)*0.9)
 Xtrain, Xval = X.iloc[:train], X.iloc[train:]
ytrain, yval = y.iloc[:train], y.iloc[train:]<init_hyperparams>
pred_y = meta_estimator.predict(pred_X)
 target_names = ['DEAD', 'SURVIVED']
print(classification_report(val_y, pred_y,target_names=target_names))
params={'random_state': 33,'n_estimators':5000,
'min_data_per_group': 5,
'boosting_type': 'gbdt',
'num_leaves': 256,
'max_dept': -1,
'learning_rate': 0.02,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'split',
'metric': 'rmse',
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}<train_on_grid>
models = ['RandomForestClassifier', 'VotingClassifier', 'MetaEstimator']
print('Best model = ', models[best_model] )
N_FOLDS = 5
kf = KFold(n_splits = N_FOLDS)
 oof = np.zeros(len(y))
oof_vanilla = np.zeros(len(y))
preds = np.zeros(len(Xtest))
params['learning_rate'] = 0.005
params['num_iterations'] = 5000
for train_ind, test_ind in tqdm(kf.split(X)) :
Xtrain = X.iloc[train_ind]
Xval = X.iloc[test_ind]
ytrain = y.iloc[train_ind]
yval = y.iloc[test_ind]
model = LGBMRegressor(**params)
 vanilla_model = LGBMRegressor()
model.fit(Xtrain, ytrain, eval_set =(( Xval,yval)) , early_stopping_rounds = 50, verbose = 0)
 vanilla_model.fit(Xtrain, ytrain)
 p = model.predict(Xval)
 p_vanilla = vanilla_model.predict(Xval)
 oof[test_ind] = p
oof_vanilla[test_ind] = p_vanilla
preds += model.predict(Xtest)/N_FOLDS
print(f'mean square error on training data(vanilla model): {np.round(mean_squared_error(y, oof_vanilla, squared=False),5)}')
 print(f'mean square error on training data(with optuna tuning): {np.round(mean_squared_error(y, oof, squared=False),5)}' )<load_from_csv>
if best_model == 0:
predictions = rfc.predict(scl_tX)
 elif best_model == 1:
predictions = vclf.predict(scl_tX)
 else:
predictions = meta_estimator.predict(test_X)
 PassengerId = test_df['PassengerId'].values
results = pd.DataFrame({ 'PassengerId': PassengerId, 'Survived': predictions })
 results.to_csv('results.csv', index=False )