Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<import_modules>
print(os.listdir(".. /input"))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 targets = train['target']
train = train.drop(columns=['id', 'target'])
 ids = test['id']
test = test.drop(columns=['id'] )
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
transformer = RobustScaler().fit(test.values)
 trans_data = transformer.transform(train.values)
 trans_test = transformer.transform(test.values)
 trans_data_df = pd.DataFrame(trans_data, columns=train.columns)
 trans_test_df = pd.DataFrame(trans_test, columns=test.columns )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
center_point_0 = trans_data_df[targets == 0].mean()
center_point_1 = trans_data_df[targets == 1].mean()
start = np.array([center_point_0, center_point_1])
 kmeans = KMeans(n_clusters=2, init=start ).fit(test.values)
 g_mix = GaussianMixture(n_components=2 ).fit(test.values)
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
predicts = kmeans.predict(train.values)
 g_predicts= g_mix.predict(train.values)
 count= 0
for i in range(len(predicts)) :
if targets[i] == predicts[i]:
count+=1
print("K-Means Cluster Matches Target : ", str(count), " times out of 250")
 data = pd.DataFrame(trans_data.copy() , columns=train.columns)
 data['k_cluster'] = predicts
data['g_mix'] = g_predicts
t_data = pd.DataFrame(trans_test.copy() , columns=test.columns)
 t_data['k_cluster'] = kmeans.predict(test.values)
 t_data['g_mix'] = g_mix.predict(test.values )
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
count=0
for i in range(len(g_predicts)) :
if targets[i] == g_predicts[i]:
count+=1
print("For Gaussian-Mixture : ", count )
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"] )<categorify>
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data.drop(columns=['k_cluster', 'g_mix'])
 clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data.drop(columns=['k_cluster', 'g_mix'] ).values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('scale_only_output.csv',index=None )
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data.values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('scale_and_clustering_output.csv',index=None )
<prepare_x_and_y><EOS> 
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data[labels]
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data[labels].values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('reduced_scale_and_clustering_output.csv',index=None )