Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<choose_model_class>
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os
GBoost = GradientBoostingRegressor(n_estimators=200, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
nthread = -1)
<choose_model_class>
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
train_EB5 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB5.npy')
 test_EB5 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB5.npy')
 train_EB6 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB6.npy')
 test_EB6 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB6.npy')
 train_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB7.npy')
 test_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB7.npy' )
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
X_train_5, X_val_5, y_train, y_val = train_test_split(train_EB5, target, test_size=0.1, random_state=42)
 X_train_6, X_val_6, _, _ = train_test_split(train_EB6, target, test_size=0.1, random_state=42)
 X_train_7, X_val_7, _, _ = train_test_split(train_EB7, target, test_size=0.1, random_state=42 )
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
%%time
lr = LogisticRegression(C=0.012, max_iter=10000)
 lr.fit(X_train_5, y_train )
score = rmsle_cv(model_lgb)
 print("lightfm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
val_preds_5 = lr.predict_proba(X_val_5)[:,1]
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
roc_auc_score(y_val, val_preds_5 )
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
log_loss(y_val, val_preds_5 )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
test_preds_5 = lr.predict_proba(test_EB5)[:,1]
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<predict_on_test>
%%time
lr = LogisticRegression(C=0.015, max_iter=10000)
 lr.fit(X_train_6, y_train )
averaged_models.fit(train.values, y_train)
 y_pre_averaged_models = averaged_models.predict(test.values)
<predict_on_test>
val_preds_6 = lr.predict_proba(X_val_6)[:,1]
test_preds_6 = lr.predict_proba(test_EB6)[:,1]
stacked_averaged_models.fit(train.values, y_train)
 y_pre_stacked_averaged_models = stacked_averaged_models.predict(test.values)
<predict_on_test>
%%time
lr = LogisticRegression(C=0.012, max_iter=10000)
 lr.fit(X_train_7, y_train )
model_xgb.fit(train.values, y_train)
 xgb_pred = model_xgb.predict(test.values)
<predict_on_test>
val_preds_7 = lr.predict_proba(X_val_7)[:,1]
test_preds_7 = lr.predict_proba(test_EB7)[:,1]
model_lgb.fit(train.values, y_train)
 lgb_pred = model_lgb.predict(test.values)
<define_variables>
roc_auc_score(y_val, 0.6*val_preds_6+0.4*val_preds_5 )
ensemble = y_pre_averaged_models*0.5 + y_pre_stacked_averaged_models *0.5 
<create_dataframe>
log_loss(y_val, 0.6*val_preds_6+0.4*val_preds_5 )
pre_df = pd.DataFrame()
pre_df["SalePrice"] = np.exp(ensemble)-1
pre_df = pre_df[["SalePrice"]] 
<create_dataframe>
roc_auc_score(y_val, 0.55*val_preds_7+0.33*val_preds_6+0.12*val_preds_5 )
submission = pd.DataFrame()
submission['Id'] = test_ID
submission = pd.concat([submission['Id'],pre_df["SalePrice"]],axis=1)
 submission<save_to_csv>
log_loss(y_val, 0.55*val_preds_7+0.33*val_preds_6+0.12*val_preds_5 )
submission.to_csv('1.csv',index=False )<set_options>
sample_submission['id'] = ids
sample_submission['label'] = 0.6*test_preds_6+0.4*test_preds_5
sample_submission.to_csv('submission_1.csv', index=False)
 sample_submission.head()
<load_from_csv><EOS> 
sample_submission['id'] = ids
sample_submission['label'] = 0.55*test_preds_7+0.33*test_preds_6+0.12*test_preds_5
sample_submission.to_csv('submission_2.csv', index=False)
 sample_submission.head()