Kaggle competition: Don't Overfit! II
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV, cross_val_score
from sklearn.feature_selection import RFE
import xgboost as xgb
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 test = pd.read_csv('.. /input/test.csv' )
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
y_train = train['target']
X_train = train.drop('target', axis=1)
 X_test = test.drop('id', axis = 1 )
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
std = StandardScaler()
X_train = std.fit_transform(X_train)
 X_test = std.fit_transform(X_test )
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
train_df['num_exclamations'] = train_df['description'].apply(lambda x: len(x.split('!')))
 test_df['num_exclamations'] = test_df['description'].apply(lambda x: len(x.split('!')))
 features_to_use.extend(["num_photos", "num_features", "num_description_words",
"created_year", "created_month", "created_day", "created_hour",
"listing_id"] )<feature_engineering>
xgb_model = xgb.XGBClassifier(random_state=42)
 param_grid = {'objective':['binary:logistic'],
'learning_rate': [0.001,0.05,0.1, 10],
'max_depth': [2,3,4,5,6],
'min_child_weight': [11],
'subsample': [0.8],
'colsample_bytree': [0.7],
'n_estimators': [1000]}
grid = GridSearchCV(estimator = xgb_model, cv=5, param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1, refit=True)
 grid.fit(X_train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
best_parameters = grid.best_params_
train_df["price_t"] = train_df["price"] / train_df["bedrooms"]
test_df["price_t"] = test_df["price"] / test_df["bedrooms"]
train_df["room_dif"] = train_df["bedrooms"] - train_df["bathrooms"]
train_df["room_sum"] = train_df["bedrooms"] + train_df["bathrooms"]
train_df["price_t1"] = train_df["price"] / train_df["room_sum"]
test_df["room_dif"] = test_df["bedrooms"] - test_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"] + test_df["bathrooms"]
test_df["price_t1"] = test_df["price"] / test_df["room_sum"]<concatenate>
xgb_model = xgb.XGBClassifier(**best_parameters)
 xgb_model.fit(X_train,y_train)
 selector = RFE(xgb_model, 200, step=1)
 selector.fit(X_train,y_train)
 xgb_preds = selector.predict_proba(X_test)[:,1]
train_predict = selector.predict(X_train)
 roc_auc_score(y_train, train_predict )
features_to_use.extend(["price_t", 'room_dif', 'room_sum', 'price_t1'] )<categorify>
lr = LogisticRegression(random_state=42)
 param_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100],
'solver': ['saga']}
grid = GridSearchCV(estimator = lr, cv=5, param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(X_train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
best_parameters = grid.best_params_
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
lr = LogisticRegression(**best_parameters)
 lr.fit(X_train,y_train)
 selector = RFE(lr, 200, step=1)
 selector.fit(X_train,y_train)
 lr_preds = lr.predict_proba(X_test)[:,1]
train_predict = lr.predict(X_train)
 roc_auc_score(y_train, train_predict )
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
final_preds =(lr_preds * 0.8 + xgb_preds * 0.2)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = final_preds
submission.to_csv('submission.csv', index=False)
 submission.head(20 )