Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<predict_on_test>
warnings.filterwarnings("ignore")
 pd.set_option('max_columns',100)
 pd.set_option('max_rows',100 )
op_rf=rfc1.predict(test_df )<create_dataframe>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train = reduce_mem_usage(train )
op=pd.DataFrame(test['PassengerId'])
 op['Survived']=op_rf<save_to_csv>
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test = reduce_mem_usage(test )
op.to_csv("Submission.csv", index=False )<define_variables>
sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 sub.head(3 )
show_graphs = True
add_interactions = False
model_tuning = False
feature_selection = False<define_variables>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
used_features = {
'PassengerId': 0,
'Pclass': 1,
'Name': 0,
'LastName': 0,
'Title': 0,
'Sex': 1,
'Sex-female x Pclass-1-2': 1,
'Sex-male x Pclass-3': 0,
'SibSp': 0,
'Parch': 0,
'FamilySize': 0,
'FamilySizeBin': 0,
'IsAloneF': 0,
'Age': 0,
'AgeBin': 1,
'IsChild': 0,
'IsChild x Pclass-1-2': 1,
'Cabin': 0,
'HasCabin': 0,
'CabinType': 0,
'Embarked': 0,
'Ticket': 0,
'TicketSize': 1,
'TicketSizeBin': 0,
'IsAloneT': 1,
'Fare': 0,
'FareOrig': 0,
'FareBin': 1,
'NameFareSize': 0,
'Group': 0,
'GroupSurvived': 1,
'GroupSize': 0
}<set_options>
target = train['SalePrice']
del train['SalePrice']
%matplotlib inline 
<load_from_csv>
train = reduce_mem_usage(train )
df_train = pd.read_csv('.. /input/train.csv')
 df_valid = pd.read_csv('.. /input/test.csv')
 yt = df_train['Survived']
df_train['Data'] = 'T'
df_valid['Data'] = 'V'
df_full = pd.concat([df_train, df_valid], sort=False, ignore_index=True)
 mask_train = df_full['Data'] == 'T'
mask_valid = df_full['Data'] == 'V'<concatenate>
train = train.fillna(-1)
 test = test.fillna(-1 )
catgroup(df_full.loc[mask_train], 'Pclass', 'Survived' )<feature_engineering>
X = train
z = target
df_full['LastName'] = df_full['Name'].str.extract('^([^,]+),', expand=False )<feature_engineering>
Xtrain, Xval, Ztrain, Zval = train_test_split(X, z, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False )
df_full['Title'] = df_full['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 dict_replace = {
"Capt": "Officer",
"Col": "Officer",
"Countess": "Noble",
"Don": "Noble",
"Dona": "Noble",
"Dr": "Noble",
"Jonkheer": "Noble",
"Lady": "Noble",
"Major": "Officer",
"Mlle": "Miss",
"Mme": "Mrs",
"Ms": "Miss",
"Rev": "Noble",
"Sir": "Noble",
}
df_full['Title'] = df_full['Title'].replace(dict_replace )<concatenate>
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'num_leaves': 45,
'learning_rate': 0.01,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=5000,
early_stopping_rounds=50,verbose_eval=50, valid_sets=valid_set )
catgroup(df_full.loc[mask_train], 'Sex', 'Survived' )<categorify>
pred_lgb = modelL.predict(test )
df_full['Sex'] = df_full['Sex'].map({'male': 0, 'female': 1} )<data_type_conversions>
feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 feature_score['score_lgb'] = modelL.feature_importance()
mask_female = df_full['Sex'] == 1
mask_class12 = df_full['Pclass'].isin([1, 2])
 df_full['Sex-female x Pclass-1-2'] =(mask_female & mask_class12 ).astype(int )<concatenate>
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
catgroup(df_full.loc[mask_train], 'Sex-female x Pclass-1-2', 'Survived' )<data_type_conversions>
parms = {'max_depth':8,
'objective':'reg:squarederror',
'eta' :0.3,
'subsample':0.8,
'lambda ' :4,
'colsample_bytree ':0.9,
'colsample_bylevel':1,
'min_child_weight': 10}
modelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,
early_stopping_rounds=30, maximize=False,
verbose_eval=10)
 print('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))
mask_male = df_full['Sex'] == 0
mask_class3 = df_full['Pclass'] == 3
df_full['Sex-male x Pclass-3'] =(mask_male & mask_class3 ).astype(int )<concatenate>
pred_xgb = modelx.predict(xgb.DMatrix(test))
catgroup(df_full.loc[mask_train], 'Sex-male x Pclass-3', 'Survived' )<feature_engineering>
feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
feature_score
df_full['FamilySize'] = df_full['SibSp'] + df_full['Parch'] + 1<concatenate>
train2 = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(train),
columns=train.columns,
index=train.index
)
catgroup(df_full[mask_train], 'FamilySize', 'Survived' )<categorify>
test2 = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(test),
columns=test.columns,
index=test.index
)
df_full['FamilySizeBin'] = df_full['FamilySizeBin'].map({'alone': 0, 'normal': 1, 'big': 2} )<data_type_conversions>
pred_lin = linreg.predict(test2 )
df_full['IsAloneF'] =(df_full['FamilySizeBin'] == 0 ).astype(int)
 catgroup(df_full.loc[mask_train], 'IsAloneF', 'Survived' )<groupby>
eli5.show_weights(linreg )
mask_noage = df_full['Age'].isnull()
df_noage = df_full.loc[mask_noage]
df_noage.groupby(['Title', 'Pclass'], as_index=False)['Name'].count()<feature_engineering>
coeff_linreg["score_linreg"] = coeff_linreg["score_linreg"].abs()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature')
 feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score
df_medians = df_full.groupby('Title')['Age'].median()
for idx, median in df_medians.iteritems() :
mask_group = df_full['Title'] == idx
df_full.loc[mask_group & mask_noage, 'Age'] = median<data_type_conversions>
feature_score.sort_values('mean', ascending=False )
df_full['IsChild'] =(df_full['Age'] < 16 ).astype(int)
 catgroup(df_full.loc[mask_train], 'IsChild', 'Survived' )<data_type_conversions>
feature_score.sort_values('total', ascending=False )
mask_class12 = df_full['Pclass'].isin([1, 2])
 df_full['IsChild x Pclass-1-2'] = df_full['IsChild'] * mask_class12.astype(int)
 catgroup(df_full.loc[mask_train], 'IsChild x Pclass-1-2', 'Survived' )<data_type_conversions>
feature_score_sort = feature_score.sort_values('total', ascending=False)
 feature_score_sort['features'] = feature_score_sort.index
feature_score_sort.columns = ['score_lgb', 'score_xgb', 'score_linreg', 'mean', 'total', 'feature']
feature_score_sort = feature_score_sort[['feature', 'score_lgb', 'score_xgb', 'score_linreg', 'mean', 'total']]
feature_score_sort
df_full['HasCabin'] = df_full['Cabin'].notnull().astype(int)
 mask_class1 =(df_full['Pclass'] == 1 ).astype(int)
 print("Correlation with 1st class: ", df_full['HasCabin'].corr(mask_class1))
catgroup(df_full.loc[mask_train], 'HasCabin', 'Survived' )<filter>
feature_score_sort.to_csv('feature_score_sort.csv', index=False )
<groupby><EOS> 
sub['SalePrice'] = pred_lgb
sub.to_csv('submission.csv', index=False )