Kaggle competition: Zillow Prize: Zillow’s Home Value Prediction (Zestimate)
<SOS> metric: ZillowMAE Kaggle data source: zillow-prize:-zillow’s-home-value-prediction-(zestimate)<create_dataframe>
%matplotlib inline
def null_verificator(data):
if data.isnull().any().any() :
view_info = pd.DataFrame(
 pd.concat(
 [data.isnull().any() ,
data.isnull().sum() ,
data.dtypes],
axis=1)
)
 view_info.columns = ['Nulos', 'Cantidad', 'Tipo Col']
size = data.shape[0]
view_info['Porcentaje'] = view_info['Cantidad'].apply(
 lambda x: str(np.round(0 if not x else x*100 / size, 2)) + ' %' 
)
 return view_info
else:
return pd.DataFrame.from_dict({'msje':"DATA LIMPIA DE NULOS"}, orient='index')
<compute_test_metric>
print('Loading Properties...')
 properties2016 = pd.read_csv('.. /input/properties_2016.csv', low_memory = False)
 properties2017 = pd.read_csv('.. /input/properties_2017.csv', low_memory = False)
 print('Loading Train...')
 train2016 = pd.read_csv('.. /input/train_2016_v2.csv', parse_dates=['transactiondate'], low_memory=False)
 train2017 = pd.read_csv('.. /input/train_2017.csv', parse_dates=['transactiondate'], low_memory=False)
 print('Loading Sample...')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', low_memory=False )
display(null_verificator(train))
display(null_verificator(test))<data_type_conversions>
def add_date_features(df):
df["transaction_year"] = df["transactiondate"].dt.year
df["transaction_month"] = df["transactiondate"].dt.month
df["transaction_day"] = df["transactiondate"].dt.day
df["transaction_quarter"] = df["transactiondate"].dt.quarter
df.drop(["transactiondate"], inplace=True, axis=1)
 return df
train['Pclass'] = train['Pclass'].astype(str)
 test['Pclass'] = test['Pclass'].astype(str )<count_values>
train2016 = add_date_features(train2016)
 train2017 = add_date_features(train2017)
 sample_submission['parcelid'] = sample_submission['ParcelId']
print('Merge Train & Test with Properties...')
 train2016 = pd.merge(train2016, properties2016, how='left', on='parcelid')
 train2017 = pd.merge(train2017, properties2017, how='left', on='parcelid')
 test_df = pd.merge(sample_submission, properties2016, how='left', on='parcelid')
 print('Concat Train 2016 & 2017...')
 train_df = pd.concat([train2016, train2017], axis=0)
 del properties2016, properties2017, train2016, train2017
gc.collect() ;
print("Train: ", train_df.shape)
 print("Test: ", test_df.shape )
for col in ['Pclass', 'Sex', 'Embarked']:
compara = pd.concat(
 [train[col].value_counts(dropna=False, normalize=True ).sort_index() ,
test[col].value_counts(dropna=False, normalize=True ).sort_index() ], axis=1 
)
 compara.columns = ['train-'+col, 'test-'+col]
display(compara)
 del compara<define_variables>
missing_perc_thresh = 0.98
exclude_missing = []
num_rows = train_df.shape[0]
for c in train_df.columns:
num_missing = train_df[c].isnull().sum()
if num_missing == 0:
continue
missing_frac = num_missing / float(num_rows)
 if missing_frac > missing_perc_thresh:
exclude_missing.append(c)
 print("We exclude: %s" % exclude_missing)
 print(len(exclude_missing))
cols_cat = ['Pclass', 'Sex', 'Embarked']
col_target = 'Survived'<create_dataframe>
exclude_unique = []
for c in train_df.columns:
num_uniques = len(train_df[c].unique())
 if train_df[c].isnull().sum() != 0:
num_uniques -= 1
if num_uniques == 1:
exclude_unique.append(c)
 print("We exclude: %s" % exclude_unique)
 print(len(exclude_unique))
mean_age_train = train.groupby(by=['Pclass', 'Sex'])['Age'].median().reset_index()
mean_age_train.columns = ['Pclass', 'Sex', 'mean_age']
mean_age_train<merge>
exclude_other = ['parcelid', 'logerror','propertyzoningdesc']
train_features = []
for c in train_df.columns:
if c not in exclude_missing \
and c not in exclude_other and c not in exclude_unique:
train_features.append(c)
 print("We use these for training: %s" % train_features)
 print(len(train_features))
train = train.merge(mean_age_train, how='left', on=['Pclass', 'Sex'])
 test = test.merge(mean_age_train, how='left', on=['Pclass', 'Sex'])
 train['Age'] = train['Age'].combine_first(train['mean_age'])
 test['Age'] = test['Age'].combine_first(train['mean_age'])
 del train['mean_age']
del test['mean_age']
train.shape, test.shape<merge>
cat_feature_inds = []
cat_unique_thresh = 1000
for i, c in enumerate(train_features):
num_uniques = len(train_df[c].unique())
 if num_uniques < cat_unique_thresh \
and not 'sqft' in c \
and not 'cnt' in c \
and not 'nbr' in c \
and not 'number' in c:
cat_feature_inds.append(i)
 print("Cat features are: %s" % [train_features[ind] for ind in cat_feature_inds] )
mean_fare_train = train.groupby(by=['Pclass', 'Sex'])['Fare'].median().reset_index()
mean_fare_train.columns = ['Pclass', 'Sex', 'mean_fare']
test = test.merge(mean_fare_train, how='left', on=['Pclass', 'Sex'])
 test['Fare'] = test['Fare'].combine_first(test['mean_fare'])
 del test['mean_fare']
train.shape, test.shape<count_missing_values>
print("Replacing NaN values by -999 !!")
 train_df.fillna(-999, inplace=True)
 test_df.fillna(-999, inplace=True )
train['Cabin'].fillna('X', inplace=True)
 test['Cabin'].fillna('X', inplace=True)
 train['Cabin'].isnull().sum() , test['Cabin'].isnull().sum()<normalization>
X_train, X_test, y_train, y_test = train_test_split(train_df[train_features], train_df.logerror, test_size=0.2, random_state=99)
 print(X_train.shape, y_train.shape)
 print(X_test.shape, y_test.shape)
 all_pool = Pool(train_df[train_features], train_df.logerror, cat_feature_inds)
 train_pool = Pool(X_train, y_train, cat_feature_inds)
 test_pool = Pool(X_test, y_test, cat_feature_inds )
train['Embarked'].fillna('S', inplace=True)
 display(null_verificator(train))
display(null_verificator(test))<feature_engineering>
catboost_parameters = {
'iterations': 400,
'learning_rate': 0.035,
'depth': 7,
'verbose': 20,
'task_type': 'GPU',
'loss_function': 'MAE',
'eval_metric': 'MAE',
'random_seed': 0,
}
train['Mr'] = train['Name'].apply(lambda _: int('Mr' in _))
test['Mr'] = test['Name'].apply(lambda _: int('Mr' in _))
train['Mrs'] = train['Name'].apply(lambda _: int('Mrs' in _))
test['Mrs'] = test['Name'].apply(lambda _: int('Mrs' in _))
train['Miss'] = train['Name'].apply(lambda _: int('Miss' in _))
test['Miss'] = test['Name'].apply(lambda _: int('Miss' in _))
train['Dr'] = train['Name'].apply(lambda _: int('Dr' in _))
test['Dr'] = test['Name'].apply(lambda _: int('Dr' in _))<feature_engineering>
model = CatBoostRegressor(**catboost_parameters)
 model.fit(train_pool, eval_set=test_pool )
train['words_in_name'] = train['Name'].apply(lambda _: len(_.split()))
test['words_in_name'] = test['Name'].apply(lambda _: len(_.split()))
view_numeric(train, 'words_in_name', col_target )<drop_column>
print_feature_importance(model, train_pool, X_train )
del train['Name']
del test['Name']<count_values>
display(train['Cabin'].value_counts())
 display(train['Ticket'].value_counts() )<groupby>
num_ensembles = 5
models = [None] * num_ensembles
for i in range(num_ensembles):
print("
Training(ensemble): %d..." %(i))
catboost_parameters['random_seed'] = i
models[i] = CatBoostRegressor(**catboost_parameters)
 models[i].fit(train_pool, eval_set=test_pool)
 print('-- Feature Importance --')
 print_feature_importance(models[i], train_pool, X_train )
train.groupby(by=['Cabin'])['PassengerId'].size()<merge>
submission = pd.DataFrame({
'ParcelId': test_df['parcelid'],
})
 test_dates = {
'201610': pd.Timestamp('2016-09-30'),
'201611': pd.Timestamp('2016-10-31'),
'201612': pd.Timestamp('2016-11-30'),
'201710': pd.Timestamp('2017-09-30'),
'201711': pd.Timestamp('2017-10-31'),
'201712': pd.Timestamp('2017-11-30')
 }
for label, test_date in test_dates.items() :
print("Predicting for: %s..." %(label))
test_df['transactiondate'] = test_date
test_df = add_date_features(test_df)
 y_pred = 0.0
for i in range(num_ensembles):
print("Ensemble:", i)
 y_pred += models[i].predict(test_df[train_features])
 y_pred /= num_ensembles
submission[label] = y_pred
submission_major = 2
print("Creating submission: submission_%03d.csv..." %(submission_major))
submission.to_csv(
 'submission_%03d.csv' %(submission_major),
float_format='%.4f',
index=False)
 print("Finished." )