Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
import numpy as np
import pandas as pd
import xgboost
import csv as csv
from xgboost import plot_importance
from matplotlib import pyplot
from sklearn.model_selection import cross_val_score,KFold
from sklearn.cross_validation import train_test_split
from sklearn.metrics import mean_absolute_error
import matplotlib.pyplot as plt
from sklearn.grid_search import GridSearchCV
from scipy.stats import skew
from collections import OrderedDict
tfv = TfidfVectorizer(min_df=3, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv<categorify>
train_dataset=pd.read_csv('.. /input/train.csv', header=0)
 test_dataset=pd.read_csv('.. /input/test.csv', header=0 )
tfv.fit(list(xtrain)+ list(xvalid))
xtrain_tfv = tfv.transform(xtrain)
 xvalid_tfv = tfv.transform(xvalid)
 xvalid_tfv.shape<compute_train_metric>
categorical_features=['MSSubClass','MSZoning','Street','Alley','LotShape','LandContour','Utilities',
'LotConfig','LandSlope','Neighborhood','Condition1','Condition2','BldgType','HouseStyle',
'RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','ExterQual','ExterCond',
'Foundation','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','Heating',
'HeatingQC','CentralAir','Electrical','KitchenQual','Functional','FireplaceQu',
'GarageType','GarageFinish','GarageQual','GarageCond','PavedDrive','PoolQC','Fence',
'MiscFeature','SaleType','SaleCondition']
every_column_except_y= [col for col in train_dataset.columns if col not in ['SalePrice','Id']]
train_dataset.describe()
clf = LogisticRegression(C=1.0)
 clf.fit(xtrain_tfv, ytrain)
 predictions = clf.predict_proba(xvalid_tfv)
 print("logloss: %0.3f " % multiclass_logloss(yvalid, predictions))<predict_on_test>
every_column_non_categorical= [col for col in train_dataset.columns if col not in categorical_features and col not in ['Id'] ]
numeric_feats = train_dataset[every_column_non_categorical].dtypes[train_dataset.dtypes != "object"].index
train_dataset[numeric_feats] = np.log1p(train_dataset[numeric_feats])
 every_column_non_categorical= [col for col in test_dataset.columns if col not in categorical_features and col not in ['Id'] ]
numeric_feats = test_dataset[every_column_non_categorical].dtypes[test_dataset.dtypes != "object"].index
test_dataset[numeric_feats] = np.log1p(test_dataset[numeric_feats] )
test_predictions = clf.predict_proba(xvalid_tfv)
 print("logloss: %0.3f " % multiclass_logloss(yvalid, predictions))<categorify>
features_with_nan=['Alley','MasVnrType','BsmtQual','BsmtQual','BsmtCond','BsmtCond','BsmtExposure',
'BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish']
def ConverNaNToNAString(data, columnList):
for x in columnList:
data[x] =str(data[x])
 ConverNaNToNAString(train_dataset, features_with_nan)
 ConverNaNToNAString(test_dataset, features_with_nan )
test_tfv = tfv.transform(df_test['text'] )<predict_on_test>
def CreateColumnPerValue(data, columnList):
for x in columnList:
values=pd.unique(data[x])
 for v in values:
column_name=x+"_"+str(v)
 data[column_name]=(data[x]==v ).astype(float)
 data.drop(x, axis=1, inplace=True)
test_predictions = clf.predict_proba(test_tfv)
 test_predictions.shape<save_to_csv>
train_dataset = pd.get_dummies(train_dataset,columns =categorical_features)
 test_dataset = pd.get_dummies(test_dataset,columns =categorical_features)
tdf = pd.DataFrame(test_predictions)
 tdf.columns = ['EAP', 'HPL', 'MWS']
submission = pd.read_csv('.. /input/spooky-author-identification/sample_submission.csv')
 result = pd.concat([submission[['id']], tdf], axis=1)
 result.head()
result.to_csv('naive_tf_idf.csv', index=False )<feature_engineering>
model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42 )
ctv = CountVectorizer(analyzer='word',token_pattern=r'\w{1,}',
ngram_range=(1, 3), stop_words = 'english')
 ctv.fit(list(xtrain)+ list(xvalid))
xtrain_ctv = ctv.transform(xtrain)
 xvalid_ctv = ctv.transform(xvalid)
<compute_train_metric>
every_column_except_y= [col for col in train_dataset.columns if col not in ['SalePrice','Id']]
model.fit(train_dataset[every_column_except_y],train_dataset['SalePrice'])
 OrderedDict(sorted(model.booster().get_fscore().items() , key=lambda t: t[1], reverse=True))
clf = LogisticRegression(C=1.0)
 clf.fit(xtrain_ctv, ytrain)
 predictions = clf.predict_proba(xvalid_ctv)
 print("logloss: %0.3f " % multiclass_logloss(yvalid, predictions))<predict_on_test>
most_relevant_features= list(dict(( k, v)for k, v in model.booster().get_fscore().items() if v >= 10 ).keys())
 print(most_relevant_features )
test_predictions = clf.predict_proba(xvalid_ctv)
 print("logloss: %0.3f " % multiclass_logloss(yvalid, predictions))<categorify>
train_dataset = train_dataset[train_dataset.GrLivArea < 8.25]
train_dataset = train_dataset[train_dataset.LotArea < 11.5]
train_dataset = train_dataset[train_dataset.SalePrice<13]
train_dataset = train_dataset[train_dataset.SalePrice>10.75]
train_dataset.drop("Id", axis=1, inplace=True )
test_ctv = ctv.transform(df_test['text'] )<predict_on_test>
train_x=train_dataset[most_relevant_features]
train_y=train_dataset['SalePrice']
best_xgb_model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 best_xgb_model.fit(train_x,train_y )
test_predictions = clf.predict_proba(test_ctv)
 test_predictions.shape<save_to_csv>
test_dataset['Prediction'] = np.expm1(best_xgb_model.predict(test_dataset[most_relevant_features]))
filename = 'submission.csv'
pd.DataFrame({'Id': test_dataset.Id, 'SalePrice': test_dataset.Prediction} ).to_csv(filename, index=False )
<compute_train_metric><EOS> 
print(test_dataset['Prediction'].head())
 print(test_dataset['Prediction'].count() )