Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<choose_model_class>
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 y_train = train['target']
X_train = train.drop('target', axis=1)
 X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.3, random_state = 40 )
eNet = ElasticNet(alpha=0,l1_ratio=0,max_iter=5 )<train_on_grid>
gridParams = {
'alpha': [0],
'max_iter':[0,5,10,15],
'l1_ratio': [0]
} 
<predict_on_test>
def fit_and_evaluate(model):
model.fit(X_train, y_train)
 model_pred = model.predict(X_test)
 model_auroc = roc_auc_score(y_test, model_pred)
 return model_auroc
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<choose_model_class>
gradient_boosted = GradientBoostingRegressor(random_state=60)
 gradient_boosted_auroc = fit_and_evaluate(gradient_boosted)
 print('Gradient Boosted Regression Performance on the test set: AUROC = %0.4f' % gradient_boosted_auroc )
averaged_models = AveragingModels(models =(KRR, model_lgb, rf, svr, model_xgb, lasso, eNet,ada, et, gbr, cat))
".format(score.mean() , score.std()))<train_model>
lr = LinearRegression()
lr_auroc = fit_and_evaluate(lr)
 print('Linear Regression Performance on the test set: AUROC = %0.4f' % lr_auroc )
averaged_models.fit(data.values,y_train )<predict_on_test>
svm = SVR(C = 1000, gamma = 0.1)
 svm_auroc = fit_and_evaluate(svm)
 print('Support Vector Machine Regression Performance on the test set: AUROC = %0.4f' % svm_auroc )
predictions = np.expm1(averaged_models.predict(test.values))<train_model>
random_forest = RandomForestRegressor(random_state=60)
 random_forest_auroc = fit_and_evaluate(random_forest)
 print('Random Forest Regression Performance on the test set: AUROC = %0.4f' % random_forest_auroc )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=8):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y.values[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<save_to_csv>
knn = KNeighborsRegressor(n_neighbors=10)
 knn_auroc = fit_and_evaluate(knn)
 print('K-Nearest Neighbors Regression Performance on the test set: AUROC = %0.4f' % knn_auroc )
sub_df.drop('y', axis = 1, inplace = True)
 sub_df['y'] = predictions
sub_df.to_csv("submission.csv", index=False)
 sub_df<set_options>
loss = ['ls', 'lad', 'huber']
n_estimators = [1500,1700, 1900, 2100, 2300]
max_depth = [2, 3, 5, 10, 15]
min_samples_leaf = [2, 4, 6, 8]
min_samples_split = [2, 4, 6, 10, 13, 16]
max_features = ['auto', 'sqrt', 'log2', None]
hyperparameter_grid = {'loss': loss,
'n_estimators': n_estimators,
'max_depth': max_depth,
'min_samples_leaf': min_samples_leaf,
'min_samples_split': min_samples_split,
'max_features': max_features}
rcParams['figure.figsize'] = 50,20
img=cv2.imread(".. /input/private-score/score.JPG")
 plt.imshow(img )<load_from_csv>
model = GradientBoostingRegressor(random_state = 42)
 random_cv = RandomizedSearchCV(estimator=model,
param_distributions=hyperparameter_grid,
cv=4, n_iter=25,
scoring = 'roc_auc',
n_jobs = -1, verbose = 1,
return_train_score = True,
random_state=42 )
train = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/train.csv')
 test = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/test.csv')
 df = train<count_unique_values>
random_cv.fit(X_train, y_train )
for i in df_cat.columns:
print('The unique values in '+i+' are: ',df[i].nunique() ,'
',df_cat[i].unique() ,'
',"--------------------------------------------------------------------------------" )<count_missing_values>
random_cv.best_estimator_
print(df.isnull().sum().sum(axis=0))<feature_engineering>
gradient_boosted = random_cv.best_estimator_
gradient_boosted_auroc = fit_and_evaluate(gradient_boosted)
 print('Gradient Boosted Regression best_estimator_ Performance on the test set: AUROC = %0.4f' % gradient_boosted_auroc )
temp=df.y.values
df_cat['y']=temp
print(df_cat.head() )<filter>
ridge_classifier = Ridge(alpha=5.0,
normalize=True,
copy_X=True,
max_iter=None,
tol=0.1,solver='svd')
 ridge_classifier_auroc = fit_and_evaluate(ridge_classifier)
 print('Ridge Performance on the test set: AUROC = %0.4f' % ridge_classifier_auroc )
print(( df.loc[df.y>150,'y'].values))
df=df[df.y<150]
print("Removing outliers based on above information and setting 150 as a threshold value.................... ")
 print(df.shape)
 df_cat=df_cat[df_cat.y<150]
df_num=df_num[df_num.y<150]<set_options>
alpha=[1.0,5,10.0,50,100,500,1000,5000,10000]
fit_intercept=[True, False]
normalize=[True, False]
copy_X=[True, False]
tol=[0.001,0.01,0.1,1,10]
solver=['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']
hyperparameter_grid = {'alpha':alpha,
'fit_intercept':fit_intercept,
'normalize':normalize,
'copy_X':copy_X,
'tol':tol,
'solver':solver
}
rcParams['figure.figsize'] = 15, 7
res = stats.probplot(df['y'], plot=plt )<define_variables>
model = Ridge()
random_cv = RandomizedSearchCV(estimator=model,
param_distributions=hyperparameter_grid,
cv=4, n_iter=25,
scoring = 'roc_auc',
n_jobs = -1, verbose = 1,
return_train_score = True,
random_state=42 )
temp = []
for i in df_num.columns:
if df[i].var() ==0:
temp.append(i)
 print(len(temp))
print(temp )<drop_column>
random_cv.fit(X_train, y_train )
count=0
low_var_col=[]
for i in test.columns:
if test[i].dtype == 'int64':
if test[i].var() <0.01:
low_var_col.append(i)
 count+=1
print(count)
 df.drop(low_var_col,axis=1,inplace=True)
 df_num.drop(low_var_col,axis=1,inplace=True)
 test.drop(low_var_col,axis=1,inplace=True )<feature_engineering>
random_cv.best_estimator_
dic={}
for i in df_num.columns:
if i!='y':
if df[i].corr(df.y)>0.25 or df[i].corr(df.y)<-0.25:
dic[i]=df[i].corr(df.y)
 print("Important Features with there respective correlations are ",'
','---------------------------------------------------------','
',dic )<define_variables>
ridge_classifier = random_cv.best_estimator_
ridge_classifier_auroc = fit_and_evaluate(ridge_classifier)
 print('ridge_classifier best_estimator_ Performance on the test set: AUROC = %0.4f' % ridge_classifier_auroc )
d = {}; done = []
cols = df.columns.values
for c in cols: d[c]=[]
for i in range(len(cols)) :
if i not in done:
for j in range(i+1, len(cols)) :
if all(df[cols[i]] == df[cols[j]]):
done.append(j)
 d[cols[i]].append(cols[j])
 dub_cols = []
for k in d.keys() :
if len(d[k])> 0:
dub_cols += d[k]
print('Dublicates:','
', dub_cols )<drop_column>
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 y_train = train['target']
X_train = train.drop('target', axis=1)
 test = pd.read_csv('.. /input/test.csv')
 X_test = test.drop('id', axis = 1)
 submission = pd.read_csv('.. /input/sample_submission.csv' )
df.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True )<drop_column>
clf = Ridge(alpha=10000, copy_X=True, fit_intercept=True, max_iter=None,
normalize=False, random_state=None, solver='sag', tol=0.001 ).fit(X_train, y_train)
<categorify><EOS> 
submission['target'] = clf.predict(X_test)
 submission.to_csv('submission.csv', index=False )