Kaggle competition: Property price prediction challenge
<SOS> metric: rmsle Kaggle data source: property-price-prediction-challenge<predict_on_test>
c_features = [
"LATITUDE",
"LONGITUDE",
"AYB",
"CMPLX_NUM",
"EYB",
"YR_RMDL",
"LIVING_GBA",
"LANDAREA",
"BATHRM",
"HF_BATHRM",
"HEAT",
"AC",
"ROOMS",
"BEDRM",
"FIREPLACES",
"SALEDATE",
"QUALIFIED",
"USECODE",
"SALE_NUM",
]
meta_y = meta_model.predict(feature_data)
 meta_y = np.exp(meta_y )<save_to_csv>
r_features = [
"LATITUDE",
"LONGITUDE",
"AYB",
"EYB",
"YR_RMDL",
"STORIES",
"GBA",
"LANDAREA",
"BATHRM",
"HF_BATHRM",
"HEAT",
"AC",
"ROOMS",
"BEDRM",
"FIREPLACES",
"KITCHENS",
"STYLE",
"STRUCT",
"GRADE",
"CNDTN",
"INTWALL",
"EXTWALL",
"ROOF",
"NUM_UNITS",
"SALEDATE",
"QUALIFIED",
"USECODE",
"SALE_NUM",
]
submission = pd.DataFrame(
 {'key': test_data_key, 'fare_amount': meta_y},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False)
 meta_y<load_from_csv>
pd.set_option("max_columns", 100)
 train = pd.read_csv(".. /input/property-price-prediction-challenge/train_set.csv", low_memory=False)
 test = pd.read_csv(".. /input/property-price-prediction-challenge/test_set.csv", low_memory=False )
df = pd.read_csv('.. /input/train.csv',nrows =2000000)
 test_set = pd.read_csv(".. /input/test.csv" )<randomize_order>
def split_by_source(X):

 is_condominium =(X["SOURCE"] == "Condominium")
 X_c = X.loc[is_condominium, :]
X_r = X.loc[~is_condominium, :]
return X_c, X_r
def preprocess_c(df, c_features):

 df = _convert_date_to_int(df)
 df = _fill_na(df)
 obj_cols = df.select_dtypes("object" ).columns.to_list()
obj_cols += ["CMPLX_NUM"]
df = _convert_object_to_category(df, obj_cols)
 if "PRICE" in df:
X, y = _split_Xy(df, c_features, y="PRICE")
 y = _preprocess_y(y)
 return X, y
else:
X = df.loc[:, c_features]
return X
def preprocess_r(df, r_features):

 df = _convert_date_to_int(df)
 df = _fill_na(df)
 obj_cols = df.select_dtypes("object" ).columns.to_list()
df = _convert_object_to_category(df, obj_cols)
 if "PRICE" in df:
X, y = _split_Xy(df, r_features, y="PRICE")
 y = _preprocess_y(y)
 return X, y
else:
X = df.loc[:, r_features]
return X
def _convert_date_to_int(X):

 X.loc[X["SALEDATE"].isnull() , "SALEDATE"] = X["SALEDATE"].mode() [0]
X.loc[:, "SALEDATE"] = pd.to_datetime(X["SALEDATE"] ).astype(np.int64)/ 10**9
return X
def _fill_na(X):

 for f in X.columns:
if X.dtypes[f] == 'object':
X.loc[:,f] = X[f].fillna("None")
 else:
X.loc[:,f] = X[f].fillna(-1)
 return X
def _convert_object_to_category(X, obj_cols):

 for col in obj_cols:
X.loc[:, col] = X[col].astype("category")
 return X
def _split_Xy(df, features, y="PRICE"):

 y = df.loc[:, y]
X = df.loc[:, features]
return X, y
def _preprocess_y(y):
y = np.log1p(y)
 return y
df = df.round(2)
 df = df.reindex(np.random.permutation(df.index)) 
<feature_engineering>
train_c, train_r = split_by_source(train)
 test_c, test_r = split_by_source(test)
 X_train_c, y_train_c = preprocess_c(train_c, c_features)
 X_test_c = preprocess_c(test_c, c_features)
 X_train_r, y_train_r = preprocess_r(train_r, r_features)
 X_test_r = preprocess_r(test_r, r_features)
def distance(lat1, lon1, lat2, lon2):
distance = np.abs(lat2 - lat1)+ np.abs(lon2 - lon1)
 return distance 
<filter>
%%time
parameter_range = {
"n_estimators": [1500],
"learning_rate": [0.05],
"min_child_samples": [3, 5],
"max_depth": [5, 7],
"num_leaves": [2**5, 2**7-50],
"colsample_bytree": [0.5, 0.7],
"lambda_l1": [0.0, 0.1, 0.5],
"lambda_l2": [0.0, 0.1, 0.5],
"n_jobs": [-1],
"random_state": [42],
}
cv_c = GridSearchCV(estimator = lgb.LGBMRegressor() ,
param_grid = parameter_range,
cv = 3, verbose = 1,
scoring = 'neg_mean_squared_error')
 cv_c.fit(X_train_c, y_train_c)
 print("best_score:")
 print(cv_c.best_score_)
 print("best_params:")
 print(cv_c.best_params_)
 cv_r = GridSearchCV(estimator = lgb.LGBMRegressor() ,
param_grid = parameter_range,
cv = 3, verbose = 1,
scoring = 'neg_mean_squared_error')
 cv_r.fit(X_train_r, y_train_r)
 print("best_score:")
 print(cv_r.best_score_)
 print("best_params:")
 print(cv_r.best_params_)
BB =(-75, -73, 40, 41.5)
 def select_within_boundingbox(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3])
 print('Old size: %d' % len(df))
df = df[select_within_boundingbox(df, BB)]
df = df[(df.passenger_count > 0)&(df.passenger_count <= 8)]
df = df[(df.fare_amount>0)&(df.fare_amount<=250)]
print('New size: %d' % len(df))<feature_engineering>
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'])
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
dataset['year'] = dataset.pickup_datetime.dt.year
return dataset
df = add_datetime_info(df)
 test_set = add_datetime_info(test_set)
<feature_engineering>
%%time
es_c = cv_c.best_estimator_
X_train_c_, X_eval_c, y_train_c_, y_eval_c = train_test_split(
 X_train_c, y_train_c, test_size=0.1, random_state=42)
 es_c.n_estimators = 100_000
es_c.fit(X_train_c_, y_train_c_,
eval_metric="rmse",
eval_set=[(X_eval_c, y_eval_c)],
early_stopping_rounds=100,
verbose=False)
 print("best scores:", dict(es_c.best_score_["valid_0"]))
print("best n_estimators:", es_c.best_iteration_)
 es_r = cv_r.best_estimator_
X_train_r_, X_eval_r, y_train_r_, y_eval_r = train_test_split(
 X_train_r, y_train_r, test_size=0.1, random_state=42)
 es_r.n_estimators = 100_000
es_r.fit(X_train_r_, y_train_r_,
eval_metric="rmse",
eval_set=[(X_eval_r, y_eval_r)],
early_stopping_rounds=100,
verbose=False)
 print("best scores:", dict(es_r.best_score_["valid_0"]))
print("best n_estimators:", es_r.best_iteration_)
<sort_values>
def predict_and_add_index(model, X):

 y_pred = model.predict(X)
 y_pred = np.expm1(y_pred ).astype(int)
 return pd.Series(y_pred, index=X.index)
 y_pred_c = predict_and_add_index(es_c, X_eval_c)
 y_pred_r = predict_and_add_index(es_r, X_eval_r)
 y_pred = pd.concat([y_pred_r, y_pred_c] ).sort_index()
y_true = pd.concat([y_eval_r, y_eval_c] ).sort_index()
y_true = np.expm1(y_true ).astype(int )
cor_frame = df.corr()
cor_frame.sort_values('fare_amount',axis=0 )<split>
def rmsle(y_true, y_pred):
error =(np.log(y_true)- np.log(y_pred))
return np.sqrt(np.mean(error**2))
print(f"RMSLE: {rmsle(y_true, y_pred):,.3f}" )
train = df.drop(['key','fare_amount','pickup_datetime'],axis=1)
 test = df['fare_amount']
X_train, X_test, y_train, y_test = train_test_split(train, test, test_size = 0.2 )<normalization>
eval_df = pd.concat([X_eval_r, X_eval_c] ).sort_index()
eval_df["y_pred"] = y_pred
eval_df["y_true"] = y_true
eval_df["abs_error"] =(eval_df["y_true"] - eval_df["y_pred"] ).abs().astype(int )
<compute_train_metric>
%%time
model_c = lgb.LGBMRegressor(**cv_c.best_params_)
 model_c.n_estimators = es_c.best_iteration_
model_c.fit(X_train_c, y_train_c)
 model_r = lgb.LGBMRegressor(**cv_r.best_params_)
 model_r.n_estimators = es_r.best_iteration_
model_r.fit(X_train_r, y_train_r)
dtrain = xgb.DMatrix(X_train, label=y_train)
 dtest = xgb.DMatrix(X_test)
 params = {'eval_metric': 'rmse',
'max_depth': 30,
'subsample': 0.8,
'eta': 0.1,
'gamma': 1,
'colsample_bytree': 1}
def xgb_evaluate(max_depth, gamma, colsample_bytree):
cv_result = xgb.cv(params, dtrain, num_boost_round=100, nfold=3)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]

 model2 = xgb.train(params, dtrain, num_boost_round=200)
<load_pretrained>
%%time
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
scoring = {
"r2": "r2",
"RMSE":make_scorer(rmse)
 }
print("
scores = cross_validate(model_c, X_train_c, y_train_c, cv = 10, scoring = scoring, return_train_score=False)
 for key,value in scores.items() :
print("{}:{:.3g}+/-{:.3g}".format(key, value.mean() , value.std()))
print("
scores = cross_validate(model_r, X_train_r, y_train_r, cv = 10, scoring = scoring, return_train_score=False)
 for key,value in scores.items() :
print("{}:{:.3g}+/-{:.3g}".format(key, value.mean() , value.std()))
import pickle 
<predict_on_test>
y_pred_c = predict_and_add_index(model_c, X_test_c)
 y_pred_r = predict_and_add_index(model_r, X_test_r)
 y_pred = pd.concat([y_pred_r, y_pred_c] ).sort_index()
predictions = model2.predict(dtest)
 predictions_train = model2.predict(dtrain )<compute_test_metric>
output = pd.DataFrame({
"Id": test["Id"],
"PRICE": pd.concat([y_pred_r, y_pred_c] ).sort_index()
})
 output.to_csv("output_ml_only.csv", index=False)
 output.head()
rmse = math.sqrt(mean_squared_error(y_test,predictions))
rmse_train = math.sqrt(mean_squared_error(y_train,predictions_train))
print(rmse)
 print(rmse_train)
<choose_model_class>
eval_df.sort_values("abs_error" ).loc[:, cols].tail(10 )
<feature_engineering>
eval_df.query("CMPLX_NUM == 1657 and SALEDATE == 1176163200")["y_true"].value_counts()
test_set_features = test_set.drop(['key','pickup_datetime'],axis=1)
 test_set_key = test_set['key']
dtest_1 = xgb.DMatrix(test_set_features)
 y_pred_reg = model2.predict(dtest_1 )<save_to_csv>
class RuleBasedRegressor:

 def __init__(self, y="PRICE", min_y=10_000_000):
self.y = y
self.min_y = min_y
def fit(self, X, y):
self.features_ = X.columns
self.data = X.copy()
if max(y)< 1000:
self.data[self.y] = np.expm1(y)
 else:
self.data[self.y] = y
is_expensive =(self.data[self.y] > self.min_y ).to_numpy()
is_condominium =(self.data["CMPLX_NUM"] != 0 ).to_numpy()
self.data = self.data.loc[is_expensive & is_condominium, :].drop_duplicates().reset_index(drop=True)
 def predict(self, X):

 n = X.shape[0]
predicts = np.zeros(n)
 for _, row in self.data.iterrows() :
is_all_equal = np.array([True]*n)
 for col in self.features_:
is_all_equal &=(X[col].to_numpy() == row[col])
 predicts[is_all_equal] = row[self.y]
return predicts
submission = pd.DataFrame(
 {'key': test_set_key, 'fare_amount': y_pred_reg},
columns = ['key', 'fare_amount'])
 submission = submission.round({'fare_amount':2})
 submission.to_csv('submission_combo.csv', index = False)
 print(submission.head())
 print("Submitted" )<set_options>
reg = RuleBasedRegressor()
X_train = pd.concat([X_train_r, X_train_c] ).sort_index()
y_train = pd.concat([y_train_r, y_train_c] ).sort_index()
X_test = pd.concat([X_test_r, X_test_c] ).sort_index()
features = ["CMPLX_NUM", "SALEDATE"]
reg.fit(X_train[features], y_train)
 y_rule = reg.predict(X_test)
 is_predicted =(y_rule > 0 )
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
y_pred = output["PRICE"].to_numpy()
y_pred[is_predicted]
from fastai.imports import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from sklearn.model_selection import train_test_split
import gc<set_options>
y_rule[is_predicted].astype(int )
def set_plot_sizes(sml, med, big):
plt.rc('font', size=sml)
 plt.rc('axes', titlesize=sml)
 plt.rc('axes', labelsize=med)
 plt.rc('xtick', labelsize=sml)
 plt.rc('ytick', labelsize=sml)
 plt.rc('legend', fontsize=sml)
 plt.rc('figure', titlesize=big)
 def parallel_trees(m, fn, n_jobs=8):
return list(ProcessPoolExecutor(n_jobs ).map(fn, m.estimators_))
def draw_tree(t, df, size=10, ratio=0.6, precision=0):

 s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,
special_characters=True, rotate=True, precision=precision)
 IPython.display.display(graphviz.Source(re.sub('Tree {',
f'Tree {{ size={size}; ratio={ratio}', s)))
 def combine_date(years, months=1, days=1, weeks=None, hours=None, minutes=None,
seconds=None, milliseconds=None, microseconds=None, nanoseconds=None):
years = np.asarray(years)- 1970
months = np.asarray(months)- 1
days = np.asarray(days)- 1
types =('<M8[Y]', '<m8[M]', '<m8[D]', '<m8[W]', '<m8[h]',
'<m8[m]', '<m8[s]', '<m8[ms]', '<m8[us]', '<m8[ns]')
 vals =(years, months, days, weeks, hours, minutes, seconds,
milliseconds, microseconds, nanoseconds)
 return sum(np.asarray(v, dtype=t)for t, v in zip(types, vals)
 if v is not None)
 def get_sample(df,n):

 idxs = sorted(np.random.permutation(len(df)) [:n])
 return df.iloc[idxs].copy()
def add_datepart(df, fldname, drop=True, time=False):

 fld = df[fldname]
fld_dtype = fld.dtype
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
fld_dtype = np.datetime64
if not np.issubdtype(fld_dtype, np.datetime64):
df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
 targ_pre = re.sub('[Dd]ate$', '', fldname)
 attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
if time: attr = attr + ['Hour', 'Minute', 'Second']
for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())
 df[targ_pre + 'Elapsed'] = fld.astype(np.int64)// 10 ** 9
if drop: df.drop(fldname, axis=1, inplace=True)
 def is_date(x): return np.issubdtype(x.dtype, np.datetime64)
 def train_cats(df):

 for n,c in df.items() :
if is_string_dtype(c): df[n] = c.astype('category' ).cat.as_ordered()
def apply_cats(df, trn):

 for n,c in df.items() :
if(n in trn.columns)and(trn[n].dtype.name=='category'):
df[n] = pd.Categorical(c, categories=trn[n].cat.categories, ordered=True)
 def fix_missing(df, col, name, na_dict):

 if is_numeric_dtype(col):
if pd.isnull(col ).sum() or(name in na_dict):
df[name+'_na'] = pd.isnull(col)
 filler = na_dict[name] if name in na_dict else col.median()
df[name] = col.fillna(filler)
 na_dict[name] = filler
return na_dict
def numericalize(df, col, name, max_n_cat):

 if not is_numeric_dtype(col)and(max_n_cat is None or len(col.cat.categories)>max_n_cat):
df[name] = col.cat.codes+1
def scale_vars(df, mapper):
warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)
 if mapper is None:
map_f = [([n],StandardScaler())for n in df.columns if is_numeric_dtype(df[n])]
mapper = DataFrameMapper(map_f ).fit(df)
 df[mapper.transformed_names_] = mapper.transform(df)
 return mapper
def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,
preproc_fn=None, max_n_cat=None, subset=None, mapper=None):

 if not ignore_flds: ignore_flds=[]
if not skip_flds: skip_flds=[]
if subset: df = get_sample(df,subset)
 else: df = df.copy()
ignored_flds = df.loc[:, ignore_flds]
df.drop(ignore_flds, axis=1, inplace=True)
 if preproc_fn: preproc_fn(df)
 if y_fld is None: y = None
else:
if not is_numeric_dtype(df[y_fld]): df[y_fld] = df[y_fld].cat.codes
y = df[y_fld].values
skip_flds += [y_fld]
df.drop(skip_flds, axis=1, inplace=True)
 if na_dict is None: na_dict = {}
else: na_dict = na_dict.copy()
na_dict_initial = na_dict.copy()
for n,c in df.items() : na_dict = fix_missing(df, c, n, na_dict)
 if len(na_dict_initial.keys())> 0:
df.drop([a + '_na' for a in list(set(na_dict.keys())- set(na_dict_initial.keys())) ], axis=1, inplace=True)
 if do_scale: mapper = scale_vars(df, mapper)
 for n,c in df.items() : numericalize(df, c, n, max_n_cat)
 df = pd.get_dummies(df, dummy_na=True)
 df = pd.concat([ignored_flds, df], axis=1)
 res = [df, y, na_dict]
if do_scale: res = res + [mapper]
return res
def rf_feat_importance(m, df):
return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}
).sort_values('imp', ascending=False)
 def set_rf_samples(n):

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n))
def reset_rf_samples() :

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n_samples))
def get_nn_mappers(df, cat_vars, contin_vars):
for v in contin_vars: df[v] = df[v].fillna(df[v].max() +100,)
 for v in cat_vars: df[v].fillna('
cat_maps = [(o, LabelEncoder())for o in cat_vars]
contin_maps = [([o], StandardScaler())for o in contin_vars]
return DataFrameMapper(cat_maps ).fit(df), DataFrameMapper(contin_maps ).fit(df)
<define_variables>
y_pred[is_predicted] = y_rule[is_predicted].astype(int )
PATH_INPUT = "/kaggle/input/"
PATH_WORKING = "/kaggle/working/"
PATH_TMP = "/tmp/"
NROWS=5_000_000<define_variables>
output["PRICE"] = y_pred
output.to_csv("output_with_rule.csv", index=False)
 output.head()