Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<compute_train_metric>
import numpy as np
import pandas as pd
import os
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 targets = train['target']
train = train.drop(columns=['id', 'target'])
 ids = test['id']
test = test.drop(columns=['id'])
 transformer = RobustScaler().fit(train.values)
 trans_x = transformer.transform(train.values)
 trans_test = transformer.transform(test.values )
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<save_to_csv>
scores = []
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(trans_x, targets)
 best_score = gs.best_score_
best_estimator = gs.best_estimator_
best_pca = 'N/a'
print(best_score)

pca = PCA(n_components=2)
 pca.fit(train.values)
 vals_2d = pca.transform(train.values)
 colors = ['red' if t == 1 else 'blue' for t in targets]
for i in range(len(vals_2d)) :
plt.scatter(x=vals_2d[i,0], y=vals_2d[i,1], color=colors[i])
out_df.to_csv("xgb_starter2.csv", index=False )<feature_engineering>
selector = RFECV(best_estimator, step=1, cv=5)
 selector = selector.fit(trans_x, targets)
 new_trans_x = selector.transform(trans_x)
 new_trans_test = selector.transform(trans_test)
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json')
 train_df.head()
test_df["bathrooms"].loc[19671] = 1.5
test_df["bathrooms"].loc[22977] = 2.0
test_df["bathrooms"].loc[63719] = 2.0
ulimit = np.percentile(train_df.price.values, 99)
 train_df['price'].ix[train_df['price']>ulimit] = ulimit
ulimit = np.percentile(test_df.price.values, 99)
 test_df['price'].ix[test_df['price']>ulimit] = ulimit
train_df["logprice"] = np.log(train_df["price"])
 test_df["logprice"] = np.log(test_df["price"])
 train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["pos"] = train_df.longitude.round(3 ).astype(str)+ '_' + train_df.latitude.round(3 ).astype(str)
 test_df["pos"] = test_df.longitude.round(3 ).astype(str)+ '_' + test_df.latitude.round(3 ).astype(str)
 vals = train_df['pos'].value_counts()
dvals = vals.to_dict()
train_df["density"] = train_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
test_df["density"] = test_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df['price_per_room'] = train_df['price']/train_df['room_sum']
test_df['price_per_room'] = test_df['price']/test_df['room_sum']
features_to_use=["bathrooms", "bedrooms", "price_t","room_sum","latitude","longitude","num_photos","density","logprice","num_features","num_description_words","price_per_room","listing_id"]
print(train_df['price'].head() )<train_model>
param_grid = {"C": [.01,.1],
"penalty":('l1','l2')}
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(new_trans_x, targets)
 print(gs.best_estimator_)
 print(gs.best_score_)
 print(gs.score(new_trans_x, targets))
predictions = gs.predict_proba(new_trans_test)[:,1]
<define_variables><EOS> 
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('output.csv',index=None )