Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
sns.set()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
df['distance2'] = np.sqrt(( df['pickup_pca0']-df['dropoff_pca0'])**2
+(df['pickup_pca1']-df['dropoff_pca1'])**2)
 df_test['distance2'] = np.sqrt(( df_test['pickup_pca0']-df_test['dropoff_pca0'])**2
+(df_test['pickup_pca1']-df_test['dropoff_pca1'])**2 )<feature_engineering>
training_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" )
df['log_trip_duration'] = np.log(df['trip_duration'] )<define_variables>
testing_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
NUM_VARS = ['pickup_pca0', 'pickup_pca1', 'dropoff_pca0', 'dropoff_pca1', 'month', 'hour', 'day', 'distance2']
TARGET = 'log_trip_duration'<define_variables>
print("Training data shape: ",training_data.shape)
 print("Testing data shape: ",testing_data.shape )
num_features = NUM_VARS<prepare_x_and_y>
test_id = testing_data["Id"]
X_train = df.loc[:, num_features]
y_train = df[TARGET]
X_test = df_test.loc[:, num_features]
X_train.shape, y_train.shape, X_test.shape<import_modules>
training_data.drop("Id", axis = 1, inplace = True)
 testing_data.drop("Id", axis = 1, inplace = True )
from sklearn.ensemble import RandomForestRegressor<import_modules>
print("Training data shape: ",training_data.shape)
 print("Testing data shape: ",testing_data.shape )
from sklearn.ensemble import RandomForestRegressor<train_model>
training_data["SalePrice"] = np.log1p(training_data["SalePrice"] )
m = RandomForestRegressor(n_estimators=100, min_samples_leaf=5, min_samples_split=15, max_features='auto', bootstrap=True)
 m.fit(X_train, y_train )<import_modules>
n_train = training_data.shape[0]
n_test = testing_data.shape[0]
y_train = training_data.SalePrice.iloc[1:]
all_data = pd.concat(( training_data,testing_data)).reset_index(drop = True)
 all_data.drop(['SalePrice'],axis = 1, inplace = True)
 print('All Data size: {}'.format(all_data.shape))
from sklearn.model_selection import cross_val_score<compute_train_metric>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({"Missing Ratio" : all_data_na})
 missing_data.head(30 )
cv_scores = cross_val_score(m, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error')
 cv_scores<predict_on_test>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
y_test_pred = m.predict(X_test)
 y_test_pred[:5]<save_to_csv>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
my_submission = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(y_test_pred)})
 my_submission.to_csv('submission.csv', index=False )<set_options>
all_data["Alley"] = all_data["Alley"].fillna("None" )
%matplotlib inline<load_from_csv>
all_data["Fence"] = all_data["Fence"].fillna("None" )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
train.isna().sum()<filter>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
<feature_engineering>
col =("GarageType" , "GarageFinish" , "GarageQual" ,"GarageCond")
 for c in col:
all_data[c] = all_data[c].fillna("None" )
train['trip_duration'] = np.log(train['trip_duration'].values)
<filter>
col2 =("GarageYrBlt" , "GarageArea" , "GarageCars")
 for c2 in col2:
all_data[c2] = all_data[c2].fillna(0 )
<compute_test_metric>
col3 =("BsmtFinSF1" , "BsmtFinSF2" , "BsmtUnfSF", "TotalBsmtSF" ,
"BsmtFullBath" , "BsmtHalfBath")
 for c3 in col3:
all_data[c3] = all_data[c3].fillna(0 )
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a)) 
<feature_engineering>
col4 =("BsmtQual" , "BsmtCond", "BsmtExposure" , "BsmtFinType1" , "BsmtFinType2")
 for c4 in col4:
all_data[c4] = all_data[c4].fillna("None" )
train["distance"] = train.apply(lambda row: haversine(row["pickup_latitude"], row["pickup_longitude"], row["dropoff_latitude"], row["dropoff_longitude"]), axis=1)
 test["distance"] = test.apply(lambda row: haversine(row["pickup_latitude"], row["pickup_longitude"], row["dropoff_latitude"], row["dropoff_longitude"]), axis=1)
<feature_engineering>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month 
<prepare_x_and_y>
all_data["MSZoning"] = all_data["MSZoning"].fillna(all_data["MSZoning"].mode() [0] )
y_train = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","distance"]]
X_testdata = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","distance"]]<import_modules>
all_data = all_data.drop(['Utilities'], axis = 1 )
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import SGDRegressor
from sklearn.model_selection import ShuffleSplit
import xgboost as xgb<split>
all_data["Functional"] = all_data['Functional'].fillna('Typ' )
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, random_state=42,test_size= 0.1)
<train_model>
all_data["Electrical"] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
rfr = RandomForestRegressor(n_estimators=100,min_samples_leaf=5, min_samples_split=50, max_depth=80,verbose=0,max_features="auto",n_jobs=-1)
 rfr.fit(X_train, y_train )<predict_on_test>
all_data["KitchenQual"] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
train_pred = rfr.predict(X_testdata)
<load_from_csv>
all_data["Exterior1st"] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data["Exterior2nd"] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
sample = pd.read_csv('.. /input/sample_submission.csv' )<create_dataframe>
all_data["SaleType"] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
my_submission = pd.DataFrame({'id': test.id, 'trip_duration': np.exp(train_pred)})
<save_to_csv>
all_data["MSSubClass"] = all_data['MSSubClass'].fillna("None" )
my_submission.to_csv('sub.csv', index=False )<set_options>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending = False)
 missing_data = pd.DataFrame({"Missing Ratio" : all_data_na})
 missing_data.head(30 )
%matplotlib inline
sns.set({'figure.figsize':(16,8)} )<load_from_csv>
all_data["MSSubClass"] = all_data["MSSubClass"].apply(str )
train = pd.read_csv(".. /input/nyc-taxi-duration-eda-by-nguyen-khac-bao-anh/training_data.csv")
 test = pd.read_csv(".. /input/nyc-taxi-duration-eda-by-nguyen-khac-bao-anh/testing_data.csv" )<prepare_x_and_y>
all_data["OverallCond"] = all_data["OverallCond"].astype(str )
xtrain = train.drop(['id', 'pickup_datetime', 'dropoff_datetime', 'trip_duration', 'log_trip_duration'], axis = 1 ).as_matrix()
xtest = test.drop(['id', 'pickup_datetime', ], axis = 1 ).as_matrix()
y = train['log_trip_duration'].values
del(train, test )<import_modules>
all_data["YrSold"] = all_data["YrSold"].astype(str)
 all_data["MoSold"] = all_data["MoSold"].astype(str )
from sklearn.model_selection import train_test_split, cross_val_score<split>
from sklearn.preprocessing import LabelEncoder
X_train, X_valid, y_train, y_valid = train_test_split(xtrain,y, test_size=0.2, random_state=42)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape<compute_test_metric>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold' )
def rmse(y,pred):
return np.sqrt(np.mean(np.square(np.log(np.exp(y)) -np.log(np.exp(pred)))) )<train_on_grid>
for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
<import_modules>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
import lightgbm as lgb<create_dataframe>
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
dtrain = lgb.Dataset(xtrain,y)
 del(X_train, y_train, X_valid, y_valid, xtrain,y )<init_hyperparams>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )
lgb_params = {
'learning_rate': 0.1,
'max_depth': 8,
'num_leaves': 55,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 300}<train_model>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
<predict_on_test>
train = all_data[1:n_train]
test = all_data[n_train:]
<train_model>
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import KFold, cross_val_score, train_test_split
import xgboost as xgb
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
model_lgb = lgb.train(lgb_params,
dtrain,
num_boost_round=1500 )<load_from_csv>
regressor = RandomForestRegressor(n_estimators = 100, random_state = 0)
 regressor.fit(train, y_train )
submit = pd.read_csv('.. /input/nyc-taxi-trip-duration/sample_submission.csv')
 submit.head()<predict_on_test>
Y_pred = regressor.predict(test)
 Y_pred
pred_test = np.exp(model_lgb.predict(xtest))<feature_engineering>
regressor = DecisionTreeRegressor(random_state = 0)
 regressor.fit(train, y_train )
submit['trip_duration'] = pred_test<save_to_csv>
y_pred = regressor.predict(test)
 print("Predicted price: ", y_pred )
submit.to_csv("submit_file.csv", index=False )<set_options>
ft = pd.DataFrame({"Actual" : y_train,"predict" : y_pred})
 ft
%matplotlib inline<load_from_csv>
from xgboost import XGBRegressor
df = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample = pd.read_csv('.. /input/sample_submission.csv' )<count_duplicates>
xgb_clf = XGBRegressor(n_estimators=1000, learning_rate=0.05)
 xgb_clf.fit(train, y_train )
is_dup = df.duplicated().sum()
is_null = df.isna().sum()
print("null : "+ str(is_null[::][1].sum()))
print("duplicated : "+ str(is_dup))<filter>
xgb_clf_cv = cross_val_score(xgb_clf,train, y_train, cv=10)
 print(xgb_clf_cv.mean() )
df = df.loc[df['pickup_longitude']> -74.02]
df = df.loc[df['pickup_latitude']< 40.77]<filter>
xgb_clf = XGBRegressor(n_estimators=1000, learning_rate=0.05)
 xgb_clf.fit(train, y_train )
df = df.loc[df['dropoff_longitude']> -74.02]
df = df.loc[df['dropoff_latitude']< 40.77]<filter>
xgb_predictions_test = xgb_clf.predict(test)
df = df[df['passenger_count'] > 0]
df = df[df['passenger_count'] <= 6]<feature_engineering>
ft1 = pd.DataFrame({"Actual" : y_train,"predict" : xgb_predictions_test} )
df['log_trip_duration'] = np.log(df['trip_duration'].values )<filter>
from sklearn.metrics import mean_squared_error
df = df[(df['log_trip_duration'] > 3.5)]
df = df[(df['log_trip_duration'] < 9)]<feature_engineering>
from sklearn.metrics import mean_squared_error
df['dist'] = np.sqrt(np.square(df['pickup_longitude'] - df['dropoff_longitude'])+ np.square(df['pickup_latitude'] - df['dropoff_latitude']))
df_test['dist'] = np.sqrt(np.square(df_test['pickup_longitude'] - df_test['dropoff_longitude'])+ np.square(df_test['pickup_latitude'] - df_test['dropoff_latitude']))<data_type_conversions>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )<feature_engineering>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
df['m'] = df['pickup_datetime'].dt.month
df_test['m'] = df_test['pickup_datetime'].dt.month
df['d'] = df['pickup_datetime'].dt.dayofweek
df_test['d'] = df_test['pickup_datetime'].dt.dayofweek
df['h'] = df['pickup_datetime'].dt.hour
df_test['h'] = df_test['pickup_datetime'].dt.hour<prepare_x_and_y>
ft2 = pd.DataFrame({"Actual" : y_train,"predict" : lgb_pred} )
X_train = df[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","m","h","d"]]
X_test = df_test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","m","h","d"]]
y = df["log_trip_duration"]<import_modules>
submission = pd.DataFrame({"Id" : test_id, "SalePrice" : lgb_pred} )
<split><EOS> 
submission.to_csv("sub.csv",index = False )