Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_x_and_y>
raw_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 raw_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 raw_train.head()
target = full_train_df["meter_reading"]
del full_train_df["meter_reading"]<drop_column>
columns = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Name', 'PassengerId']
train = raw_train[np.append(columns, ['Survived'])].copy().set_index('PassengerId')
 test = raw_test[columns].copy().set_index('PassengerId')
 class OneHotEncoder(SklearnOneHotEncoder):
def __init__(self, **kwargs):
super(OneHotEncoder, self ).__init__(**kwargs)
 self.fit_flag = False
def fit(self, X, **kwargs):
out = super().fit(X)
 self.fit_flag = True
return out
def transform(self, X, categories, index='', name='', **kwargs):
sparse_matrix = super(OneHotEncoder, self ).transform(X)
 new_columns = self.get_new_columns(X=X, name=name, categories=categories)
 d_out = pd.DataFrame(sparse_matrix.toarray() , columns=new_columns, index=index)
 return d_out
def fit_transform(self, X, categories, index, name, **kwargs):
self.fit(X)
 return self.transform(X, categories=categories, index=index, name=name)
 def get_new_columns(self, X, name, categories):
new_columns = []
for j in range(len(categories)) :
new_columns.append('{}_{}'.format(name, categories[j]))
return new_columns
class data_preparation() :
def __init__(self):
self.enc_sex = LabelEncoder()
self.scaler_age = StandardScaler()
self.enc_title = LabelEncoder()
self.onehotenc_title = OneHotEncoder(handle_unknown='ignore')
 self.onehotenc_pclass = OneHotEncoder(handle_unknown='ignore')
 self.enc_embarked = LabelEncoder()
self.onehotenc_embarked = OneHotEncoder(handle_unknown='ignore')
 def transform(self, data_untouched):
data = data_untouched.copy(deep=True)
 data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand = False)
 data['Embarked'] = data['Embarked'].fillna(data['Embarked'].mode() [0])
 try:
data['Sex'] = self.enc_sex.transform(data['Sex'].values)
 data['Age'] = self.scaler_age.transform(data['Age'].values.reshape(-1, 1))
data['Title'] = data['Title'].map(lambda x: self.enc_title.transform([x])[0] if x in self.enc_title.classes_ else -1)
 data['Embarked'] = data['Embarked'].map(lambda x: self.enc_embarked.transform([str(x)])[0] if x in self.enc_embarked.classes_ else -1)
 data = pd.concat([self.onehotenc_title.transform(data['Title'].values.reshape(-1,1), categories=self.enc_title.classes_,name='Title', index=data.index),
self.onehotenc_pclass.transform(data['Pclass'].values.reshape(-1,1), categories=[3,2,1], name='Pclass', index=data.index),
self.onehotenc_embarked.transform(data['Embarked'].values.reshape(-1,1), categories=self.enc_embarked.classes_, name='Embarked', index=data.index),
data],axis=1)
 except NotFittedError:
self.mean_age = data['Age'].median()
self.median_fare = data['Fare'].median()
data['Sex'] = self.enc_sex.fit_transform(data['Sex'].values)
 data['Age'] = self.scaler_age.fit_transform(data['Age'].values.reshape(-1, 1))
data['Title'] = self.enc_title.fit_transform(data['Title'].values)
 data['Embarked'] = self.enc_embarked.fit_transform(data['Embarked'].astype(str ).values)
 data = pd.concat([self.onehotenc_title.fit_transform(data['Title'].values.reshape(-1,1), categories=self.enc_title.classes_, name='Title', index=data.index),
self.onehotenc_pclass.fit_transform(data['Pclass'].values.reshape(-1,1), categories=[3,2,1], name='Pclass', index=data.index),
self.onehotenc_embarked.fit_transform(data['Embarked'].values.reshape(-1,1), categories=self.enc_embarked.classes_, name='Embarked', index=data.index),
data], axis=1)
 data['Family_size'] = data['SibSp'] + data['Parch']
data['IsAlone'] =(data['Family_size'] == 0 ).astype(int)
 data['Age'] = data['Age'].fillna(self.mean_age)
 data['Fare'] = data['Fare'].fillna(self.median_fare)
 data = data.drop(columns=['Pclass', 'Embarked', 'Title', 'Name'])
 return data
data_prep = data_preparation()
X_train_full = data_prep.transform(train.drop(columns='Survived'))
y_train_full = train['Survived']
X_test = data_prep.transform(test)
 np.random.seed(0)
 X_train, X_val, y_train, y_val = train_test_split(X_train_full, y_train_full, test_size=0.10, random_state=50 )
full_train_df.drop(drop_cols, axis=1)
 gc.collect()<choose_model_class>
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import BernoulliNB
from sklearn.ensemble import VotingClassifier
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse'},
'subsample': 0.4,
'subsample_freq': 1,
'learning_rate': 0.25,
'num_leaves': 31,
'feature_fraction': 0.8,
'lambda_l1': 1,
'lambda_l2': 1
}
folds = 2
seed = 666
kf = GroupKFold(n_splits=folds)
 models = []
for train_index, val_index in kf.split(full_train_df, full_train_df['building_id'], groups=full_train_df['group']):
train_X = full_train_df[feat_cols].iloc[train_index]
val_X = full_train_df[feat_cols].iloc[val_index]
train_y = target.iloc[train_index]
val_y = target.iloc[val_index]
lgb_train = lgb.Dataset(train_X, train_y, categorical_feature=categoricals)
 lgb_eval = lgb.Dataset(val_X, val_y, categorical_feature=categoricals)
 gbm = lgb.train(params,
lgb_train,
num_boost_round=500,
valid_sets=(lgb_train, lgb_eval),
early_stopping_rounds=100,
verbose_eval = 100)
 models.append(gbm )<drop_column>
list_params = {
'Tree':{'max_depth': np.logspace(2,8,6)},
'SVC':{'C' : np.linspace(0.01,1,3), 'kernel':['poly'], 'degree':np.linspace(2,8,4 ).astype(int)},
'KNN':{'n_neighbors' : np.linspace(2,30,29 ).astype(int)},
'gauss':{'var_smoothing' : np.logspace(-8,-10,3)},
'bern':{'alpha' : np.linspace(0.1,1,10)}
}
list_base_model = {
'Tree' : DecisionTreeClassifier(random_state=0),
'SVC' : SVC(random_state=0),
'KNN' : KNeighborsClassifier() ,
'gauss' : GaussianNB() ,
'bern' : BernoulliNB()
}
list_models = pd.DataFrame(list_params.items() , columns = ['name', 'params'] ).set_index('name')
 list_models['model'] = list_base_model.values()
list_models['grid_model'] = list_models.apply(lambda x: GridSearchCV(x['model'], x['params']), axis=1)
 list_models['grid_model'] = list_models.apply(lambda x: x['grid_model'].fit(X_train, y_train), axis=1)
 list_models['score'] = list_models.apply(lambda x: x['grid_model'].predict(X_val), axis=1, result_type='reduce' ).apply(lambda x: accuracy_score(x, y_val))
print('Score for each base model using GridSearch:')
 print(list_models['score'] )
del full_train_df, train_X, val_X, lgb_train, lgb_eval, train_y, val_y, target
gc.collect()<feature_engineering>
models =(( 'Tree', DecisionTreeClassifier(**list_models.loc['Tree', 'grid_model'].best_params_, random_state=40)) , 
('SVC', SVC(**list_models.loc['SVC', 'grid_model'].best_params_, random_state=40, probability=True)) , 
('KNN', KNeighborsClassifier(**list_models.loc['KNN', 'grid_model'].best_params_)) , 
('gauss', GaussianNB(**list_models.loc['gauss', 'grid_model'].best_params_)) , 
('bern', BernoulliNB(**list_models.loc['bern', 'grid_model'].best_params_)))
 vot = VotingClassifier(models, voting='hard')
 vot.fit(X_train,y_train)
 print('Score of the VotingClassifer with hard voting:', accuracy_score(vot.predict(X_val), y_val))
<feature_engineering>
vot = VotingClassifier(models, voting='soft')
 vot.fit(X_train,y_train)
 print('Score of the VotingClassifer with soft voting:', accuracy_score(vot.predict(X_val), y_val))
for item in beaufort:
full_test_df.loc[(full_test_df['wind_speed']>=item[1])&(full_test_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]<categorify>
vot = VotingClassifier(models, voting='soft', weights=list_models['score']/list_models['score'].sum())
 vot.fit(X_train,y_train)
 print('Score of the VotingClassifer with soft weighted voting:', accuracy_score(vot.predict(X_val), y_val))
full_test_df = encode_cyclic_feature(full_test_df, 'weekday', 7)
 full_test_df = encode_cyclic_feature(full_test_df, 'hour', 24)
 full_test_df = encode_cyclic_feature(full_test_df, 'day', 31)
 full_test_df = encode_cyclic_feature(full_test_df, 'month', 12 )<drop_column>
vot = VotingClassifier(models, voting='soft')
 vot.fit(X_train_full,y_train_full)
 submission = pd.DataFrame({'PassengerId': test.index, 'Survived': vot.predict(X_test)})
 submission.to_csv('submission_stack.csv', index=False )
full_test_df = full_test_df[feat_cols]<feature_engineering>
models =(( 'Tree', DecisionTreeClassifier(**list_models.loc['Tree', 'grid_model'].best_params_, random_state=40)) , 
('SVC', SVC(**list_models.loc['SVC', 'grid_model'].best_params_, random_state=40, probability=True)) , 
('KNN', KNeighborsClassifier(**list_models.loc['KNN', 'grid_model'].best_params_)) , 
('gauss', GaussianNB(**list_models.loc['gauss', 'grid_model'].best_params_)) , 
('bern', BernoulliNB(**list_models.loc['bern', 'grid_model'].best_params_)))
 final_estimator = MLPClassifier(hidden_layer_sizes =(100,50,), random_state=1)
 stack = StackingClassifier(models, final_estimator=final_estimator)
 stack.fit(X_train,y_train)
 print('Score of the StackingClassifer with hard voting:', accuracy_score(stack.predict(X_val), y_val))
full_test_df = reduce_mem_usage(full_test_df )<find_best_params>
models =(( 'Tree', DecisionTreeClassifier(**list_models.loc['Tree', 'grid_model'].best_params_, random_state=40)) , 
('SVC', SVC(**list_models.loc['SVC', 'grid_model'].best_params_, random_state=40, probability=True)) , 
('KNN', KNeighborsClassifier(**list_models.loc['KNN', 'grid_model'].best_params_)) , 
('gauss', GaussianNB(**list_models.loc['gauss', 'grid_model'].best_params_)) , 
('bern', BernoulliNB(**list_models.loc['bern', 'grid_model'].best_params_)))
 MLP = MLPClassifier(hidden_layer_sizes =(50,5,), random_state=1)
 stack_MLP = StackingClassifier(models, final_estimator=MLP)
 log = LogisticRegression()
stack_log = StackingClassifier(models, final_estimator=log)
 QDA = QuadraticDiscriminantAnalysis()
stack_QDA = StackingClassifier(models, final_estimator=QDA)
 small_MLP = MLPClassifier(hidden_layer_sizes =(5,), random_state=10)
 stack_final = StackingClassifier([('MLP',stack_MLP),('log', stack_log),('QDA', stack_QDA)], final_estimator=small_MLP)
 stack_final.fit(X_train,y_train)
 print('Score of the StackingClassifer:', accuracy_score(stack_final.predict(X_val), y_val))
i=0
res=[]
step_size = 50000
for j in tqdm(range(int(np.ceil(full_test_df.shape[0]/50000)))) :
res.append(np.expm1(sum([model.predict(full_test_df.iloc[i:i+step_size])for model in models])/folds))
i+=step_size<concatenate>
list_base_model = {
'Tree' : BaggingClassifier(DecisionTreeClassifier(max_depth=10), n_estimators=100, max_samples=0.5, random_state=10),
'SVC' : BaggingClassifier(SVC(C=0.2), n_estimators=20, max_samples=0.5, random_state=10),
'KNN' : BaggingClassifier(KNeighborsClassifier(n_neighbors=8, weights='distance'), n_estimators=50, max_samples=0.3, random_state=10),
'gauss' : BaggingClassifier(GaussianNB() , n_estimators=100, max_samples=0.5, random_state=10),
'bern' : BaggingClassifier(BernoulliNB(alpha=0.01, binarize=0.5), n_estimators=200, max_samples=0.4, random_state=10)
 }
list_models_bag = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_bag['model'] = list_models_bag.apply(lambda x: x['model'].fit(X_train, y_train), axis=1)
 list_models_bag['score'] = list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='reduce' ).apply(lambda x: accuracy_score(x, y_val))
print('Score for Bagging model on samples:')
 print(list_models_bag['score'] )
res = np.concatenate(res )<save_to_csv>
print('Accuracy using all model:',accuracy_score(list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='expand' ).mode().values.reshape(-1,), y_val))
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = res
submission.loc[submission['meter_reading']<0, 'meter_reading'] = 0
submission.to_csv('submission_fe_lgbm.csv', index=False)
 submission<set_options>
list_base_model = {
'Tree' : BaggingClassifier(DecisionTreeClassifier(max_depth=10), n_estimators=10, max_features=0.7, random_state=10),
'SVC' : BaggingClassifier(SVC(C=0.2), n_estimators=50, max_features=0.3, random_state=10),
'KNN' : BaggingClassifier(KNeighborsClassifier(n_neighbors=8, weights='distance'), n_estimators=10, max_features=0.7, random_state=10),
'gauss' : BaggingClassifier(GaussianNB() , n_estimators=10, max_features=0.7, random_state=10),
'bern' : BaggingClassifier(BernoulliNB(alpha=0.01, binarize=0.5), n_estimators=10, max_features=0.7, random_state=10)
 }
list_models_bag = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_bag['model'] = list_models_bag.apply(lambda x: x['model'].fit(X_train, y_train), axis=1)
 list_models_bag['score'] = list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='reduce' ).apply(lambda x: accuracy_score(x, y_val))
print('Score for Bagging model on features:')
 print(list_models_bag['score'] )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_csv>
print('Accuracy using all model:',accuracy_score(list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='expand' ).mode().values.reshape(-1,), y_val))
basePath = '/kaggle/input/ashrae-energy-prediction/'
building_metadata = pd.read_csv(basePath+'building_metadata.csv')
 weather_train = pd.read_csv(basePath+'weather_train.csv')
 raw_train = pd.read_csv(basePath+'train.csv')
 print(building_metadata.head(10))
print(weather_train.head(10))
print(raw_train.head(10))<set_options>
list_base_model = {
'Tree' : BaggingClassifier(DecisionTreeClassifier(max_depth=10), n_estimators=10, max_features=0.7, max_samples=0.5, random_state=10),
'SVC' : BaggingClassifier(SVC(C=0.2), n_estimators=50, max_features=0.4, max_samples=0.4, random_state=10),
'KNN' : BaggingClassifier(KNeighborsClassifier(n_neighbors=8, weights='distance'), n_estimators=50, max_features=0.5, max_samples=0.5, random_state=10),
'gauss' : BaggingClassifier(GaussianNB() , n_estimators=50, max_features=0.5, max_samples=0.5, random_state=10),
'bern' : BaggingClassifier(BernoulliNB(alpha=0.01, binarize=0.5), n_estimators=50, max_features=0.5, max_samples=0.5, random_state=10)
 }
list_models_bag = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_bag['model'] = list_models_bag.apply(lambda x: x['model'].fit(X_train, y_train), axis=1)
 list_models_bag['score'] = list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='reduce' ).apply(lambda x: accuracy_score(x, y_val))
print('Score for Bagging model on features:')
 print(list_models_bag['score'] )
del curSiteData
gc.collect()<drop_column>
print('Accuracy using all model:',accuracy_score(list_models_bag.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='expand' ).mode().values.reshape(-1,), y_val))
weather_train = weather_train.drop('hour',axis=1 )<feature_engineering>
list_models_bag = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_bag['model'] = list_models_bag.apply(lambda x: x['model'].fit(X_train_full, y_train_full), axis=1)
 my_submission = pd.DataFrame({'PassengerId': test.index, 'Survived': list_models_bag.apply(lambda x: x['model'].predict(X_test), axis=1, result_type='expand' ).mode().values.reshape(-1,)})
 my_submission.to_csv('submission_bag.csv', index=False )
offsetBase = weather_train[['site_id','timestamp','air_temperature']]
offsetBase['timestamp'] = pd.to_datetime(offsetBase['timestamp'])
 offsetBase['temp_rank'] = offsetBase.groupby(['site_id', offsetBase.timestamp.dt.date])['air_temperature'].rank('average' )<data_type_conversions>
list_base_model = {
'Tree' : AdaBoostClassifier(DecisionTreeClassifier(max_depth=10), n_estimators=20, random_state=0, learning_rate=1.05),
'SVC' : AdaBoostClassifier(SVC(C=0.3), algorithm='SAMME',n_estimators=5, random_state=0, learning_rate=1.1),
'gauss' : AdaBoostClassifier(GaussianNB() , n_estimators=5, random_state=0, learning_rate=1.3),
'bern' : AdaBoostClassifier(BernoulliNB(alpha=0.01, binarize=0.5), n_estimators=5, random_state=0, learning_rate=1.3)
 }
list_models_boost = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_boost['model'] = list_models_boost.apply(lambda x: x['model'].fit(X_train, y_train), axis=1)
 list_models_boost['score'] = list_models_boost.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='reduce' ).apply(lambda x: accuracy_score(x, y_val))
print('Score for Boosting model:')
 print(list_models_boost['score'] )
weather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])
 def alignTimestamp(df):
df['offset'] = df.site_id.map(site_offset)
 df['timestamp_updated'] = df['timestamp'] - pd.to_timedelta(df['offset'],unit='H')
 df['timestamp'] = df['timestamp_updated']
df = df.drop('timestamp_updated',axis=1)
 return df
weather_train = alignTimestamp(weather_train )<data_type_conversions>
print('Accuracy using all model:',accuracy_score(list_models_boost.apply(lambda x: x['model'].predict(X_val), axis=1, result_type='expand' ).mode().iloc[0,:].values.reshape(-1,), y_val))
<feature_engineering><EOS> 
list_models_boost = pd.DataFrame(list_base_model.items() , columns = ['name', 'model'] ).set_index('name')
 list_models_boost['model'] = list_models_boost.apply(lambda x: x['model'].fit(X_train_full, y_train_full), axis=1)
 my_submission = pd.DataFrame({'PassengerId': test.index, 'Survived': list_models_boost.apply(lambda x: x['model'].predict(X_test), axis=1, result_type='expand' ).mode().iloc[0,:].astype(int ).values.reshape(-1,)})
 my_submission.to_csv('submission_boost.csv', index=False )