Kaggle competition: Zillow Prize: Zillow’s Home Value Prediction (Zestimate)
<SOS> metric: ZillowMAE Kaggle data source: zillow-prize:-zillow’s-home-value-prediction-(zestimate)<split>
pd.options.display.max_columns = None
pd.options.mode.chained_assignment = None
pd.options.display.float_format
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams[ 'figure.figsize' ] = 8 , 6

 def load_training_data(file_name):
return pd.read_csv(file_name)

def load_properties_data(file_name):
def convert_true_to_float(df, col):
df.loc[df[col] == 'true', col] = '1'
df.loc[df[col] == 'Y', col] = '1'
df[col] = df[col].astype(float)
 prop = pd.read_csv(file_name, dtype={
'propertycountylandusecode': str,
'hashottuborspa': str,
'propertyzoningdesc': str,
'fireplaceflag': str,
'taxdelinquencyflag': str
})
 for col in ['hashottuborspa', 'fireplaceflag', 'taxdelinquencyflag']:
convert_true_to_float(prop, col)
 return prop

 def rename_columns(prop):
prop.rename(columns={
'parcelid': 'parcelid',
'airconditioningtypeid': 'cooling_id',
'architecturalstyletypeid': 'architecture_style_id',
'basementsqft': 'basement_sqft',
'bathroomcnt': 'bathroom_cnt',
'bedroomcnt': 'bedroom_cnt',
'buildingclasstypeid': 'framing_id',
'buildingqualitytypeid': 'quality_id',
'calculatedbathnbr': 'bathroom_cnt_calc',
'decktypeid': 'deck_id',
'finishedfloor1squarefeet': 'floor1_sqft',
'calculatedfinishedsquarefeet': 'finished_area_sqft_calc',
'finishedsquarefeet12': 'finished_area_sqft',
'finishedsquarefeet13': 'perimeter_area',
'finishedsquarefeet15': 'total_area',
'finishedsquarefeet50': 'floor1_sqft_unk',
'finishedsquarefeet6': 'base_total_area',
'fips': 'fips',
'fireplacecnt': 'fireplace_cnt',
'fullbathcnt': 'bathroom_full_cnt',
'garagecarcnt': 'garage_cnt',
'garagetotalsqft': 'garage_sqft',
'hashottuborspa': 'spa_flag',
'heatingorsystemtypeid': 'heating_id',
'latitude': 'latitude',
'longitude': 'longitude',
'lotsizesquarefeet': 'lot_sqft',
'poolcnt': 'pool_cnt',
'poolsizesum': 'pool_total_size',
'pooltypeid10': 'pool_unk_1',
'pooltypeid2': 'pool_unk_2',
'pooltypeid7': 'pool_unk_3',
'propertycountylandusecode': 'county_landuse_code',
'propertylandusetypeid': 'landuse_type_id' ,
'propertyzoningdesc': 'zoning_description',
'rawcensustractandblock': 'census_1',
'regionidcity': 'city_id',
'regionidcounty': 'county_id',
'regionidneighborhood': 'neighborhood_id',
'regionidzip': 'region_zip',
'roomcnt': 'room_cnt',
'storytypeid': 'story_id',
'threequarterbathnbr': 'bathroom_small_cnt',
'typeconstructiontypeid': 'construction_id',
'unitcnt': 'unit_cnt',
'yardbuildingsqft17': 'patio_sqft',
'yardbuildingsqft26': 'storage_sqft',
'yearbuilt': 'year_built',
'numberofstories': 'story_cnt',
'fireplaceflag': 'fireplace_flag',
'structuretaxvaluedollarcnt': 'tax_structure',
'taxvaluedollarcnt': 'tax_parcel',
'assessmentyear': 'tax_year',
'landtaxvaluedollarcnt': 'tax_land',
'taxamount': 'tax_property',
'taxdelinquencyflag': 'tax_overdue_flag',
'taxdelinquencyyear': 'tax_overdue_year',
'censustractandblock': 'census_2'
}, inplace=True)

def retype_columns(prop):
def float_to_categorical(df, col):
df[col] = df[col] - df[col].min()
df.loc[df[col].isnull() , col] = -1
df[col] = df[col].astype(int ).astype('category')
 list_float2categorical = ['cooling_id', 'architecture_style_id', 'framing_id',
'heating_id', 'county_id', 'construction_id', 'fips', 'landuse_type_id',
'county_landuse_code_id', 'zoning_description_id']
for col in prop.columns:
if col in list_float2categorical:
float_to_categorical(prop, col)
 elif prop[col].dtype.name == 'float64':
prop[col] = prop[col].astype(np.float32)
 gc.collect()

 def compute_datetime_aggregate_features(train):
dt = pd.to_datetime(train.transactiondate ).dt
train['year'] = dt.year
train['month'] = dt.month
train['quarter'] = dt.quarter
logerror_year = train.groupby('year' ).logerror.median().to_frame() \ 
.rename(index=str, columns={"logerror": "logerror_year"})
 logerror_month = train.groupby('month' ).logerror.median().to_frame() \ 
.rename(index=str, columns={"logerror": "logerror_month"})
 logerror_quarter = train.groupby('quarter' ).logerror.median().to_frame() \ 
.rename(index=str, columns={"logerror": "logerror_quarter"})
 logerror_year.index = logerror_year.index.map(int)
 logerror_month.index = logerror_month.index.map(int)
 logerror_quarter.index = logerror_quarter.index.map(int)
 train.drop(['year', 'month', 'quarter'], axis=1, errors='ignore', inplace=True)
 return logerror_year, logerror_month, logerror_quarter

 def add_datetime_aggregate_features(df, logerror_year, logerror_month, logerror_quarter):
dt = pd.to_datetime(df.transactiondate ).dt
df['year'] = dt.year
df['month'] = dt.month
df['quarter'] = dt.quarter
df = df.merge(how='left', right=logerror_year, on='year')
 df = df.merge(how='left', right=logerror_month, on='month')
 df = df.merge(how='left', right=logerror_quarter, on='quarter')
 df = df.drop(['year', 'month', 'quarter', 'transactiondate'], axis=1, errors='ignore')
 return df

 def add_simple_datetime_features(df):
dt = pd.to_datetime(df.transactiondate ).dt
df['year'] =(dt.year - 2016 ).astype(int)
 df['month'] =(dt.month ).astype(int)
 df['quarter'] =(dt.quarter ).astype(int)
 df.drop(['transactiondate'], axis=1, inplace=True)

def print_complete_percentage(df):
complete_percent = []
total_cnt = len(df)
 for col in df.columns:
complete_cnt = total_cnt -(df[col].isnull() ).sum()
complete_cnt -=(df[col] == -1 ).sum()
complete_percent.append(( col, complete_cnt * 1.00 / total_cnt))
complete_percent.sort(key=lambda x: x[1], reverse=True)
 for col, percent in complete_percent:
print("{}: {}".format(col, percent))
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )<choose_model_class>
%%time
prop_2016 = load_properties_data("/kaggle/input/zillow-prize-1/properties_2016.csv")
 prop_2017 = load_properties_data("/kaggle/input/zillow-prize-1/properties_2017.csv")
 assert len(prop_2016)== len(prop_2017)
 print("Number of properties: {}".format(len(prop_2016)))
 print("Number of property features: {}".format(len(prop_2016.columns)-1))
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 128, kernel_size =(2,2),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation = "relu"))
model.add(Dropout(0.3))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
def get_landuse_code_df(prop_2016, prop_2017):
temp = prop_2016.groupby('county_landuse_code')['county_landuse_code'].count()
landuse_codes = list(temp[temp >= 300].index)
 temp = prop_2017.groupby('county_landuse_code')['county_landuse_code'].count()
landuse_codes += list(temp[temp >= 300].index)
 landuse_codes = list(set(landuse_codes))
df_landuse_codes = pd.DataFrame({'county_landuse_code': landuse_codes,
'county_landuse_code_id': range(len(landuse_codes)) })
 return df_landuse_codes
def get_zoning_desc_code_df(prop_2016, prop_2017):
temp = prop_2016.groupby('zoning_description')['zoning_description'].count()
zoning_codes = list(temp[temp >= 5000].index)
 temp = prop_2017.groupby('zoning_description')['zoning_description'].count()
zoning_codes += list(temp[temp >= 5000].index)
 zoning_codes = list(set(zoning_codes))
df_zoning_codes = pd.DataFrame({'zoning_description': zoning_codes,
'zoning_description_id': range(len(zoning_codes)) })
 return df_zoning_codes
def process_columns(df, df_landuse_codes, df_zoning_codes):
df = df.merge(how='left', right=df_landuse_codes, on='county_landuse_code')
 df = df.drop(['county_landuse_code'], axis=1)
 df = df.merge(how='left', right=df_zoning_codes, on='zoning_description')
 df = df.drop(['zoning_description'], axis=1)
 df.loc[df.county_id == 3101, 'county_id'] = 0
df.loc[df.county_id == 1286, 'county_id'] = 1
df.loc[df.county_id == 2061, 'county_id'] = 2
df.loc[df.landuse_type_id == 279, 'landuse_type_id'] = 261
return df
rename_columns(prop_2016)
 rename_columns(prop_2017)
 df_landuse_codes = get_landuse_code_df(prop_2016, prop_2017)
 df_zoning_codes = get_zoning_desc_code_df(prop_2016, prop_2017)
 prop_2016 = process_columns(prop_2016, df_landuse_codes, df_zoning_codes)
 prop_2017 = process_columns(prop_2017, df_landuse_codes, df_zoning_codes)
 retype_columns(prop_2016)
 retype_columns(prop_2017)
 prop_2017.head()
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
train_2016 = load_training_data("/kaggle/input/zillow-prize-1/train_2016_v2.csv")
 train_2017 = load_training_data("/kaggle/input/zillow-prize-1/train_2017.csv")
 print("Number of 2016 transaction records: {}".format(len(train_2016)))
 print("Number of 2017 transaction records: {}".format(len(train_2017)))
 print("
", train_2016.head())
 print("
", train_2017.head() )
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
for prop in [prop_2016, prop_2017]:
prop['avg_garage_size'] = prop['garage_sqft'] / prop['garage_cnt']
prop['property_tax_per_sqft'] = prop['tax_property'] / prop['finished_area_sqft_calc']
prop['location_1'] = prop['latitude'] + prop['longitude']
prop['location_2'] = prop['latitude'] - prop['longitude']
prop['location_3'] = prop['latitude'] + 0.5 * prop['longitude']
prop['location_4'] = prop['latitude'] - 0.5 * prop['longitude']
prop['missing_finished_area'] = prop['finished_area_sqft'].isnull().astype(np.float32)
 prop['missing_total_area'] = prop['total_area'].isnull().astype(np.float32)
 prop.drop(['finished_area_sqft', 'total_area'], axis=1, inplace=True)
 prop['missing_bathroom_cnt_calc'] = prop['bathroom_cnt_calc'].isnull().astype(np.float32)
 prop.drop(['bathroom_cnt_calc'], axis=1, inplace=True)
 prop['derived_room_cnt'] = prop['bedroom_cnt'] + prop['bathroom_cnt']
mask =(prop.room_cnt >= 1)
 prop.loc[mask, 'avg_area_per_room'] = prop.loc[mask, 'finished_area_sqft_calc'] / prop.loc[mask, 'room_cnt']
mask =(prop.derived_room_cnt >= 1)
 prop.loc[mask,'derived_avg_area_per_room'] = prop.loc[mask,'finished_area_sqft_calc'] / prop.loc[mask,'derived_room_cnt']
prop_2017.head()
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
def add_aggregate_features(df, group_col, agg_cols):
df[group_col + '-groupcnt'] = df[group_col].map(df[group_col].value_counts())
 new_columns = []
for col in agg_cols:
aggregates = df.groupby(group_col, as_index=False)[col].agg([np.mean])
 aggregates.columns = [group_col + '-' + col + '-' + s for s in ['mean']]
new_columns += list(aggregates.columns)
 df = df.merge(how='left', right=aggregates, on=group_col)
 for col in agg_cols:
mean = df[group_col + '-' + col + '-mean']
diff = df[col] - mean
df[group_col + '-' + col + '-' + 'diff'] = diff
if col != 'year_built':
df[group_col + '-' + col + '-' + 'percent'] = diff / mean
threshold = 100
df.loc[df[group_col + '-groupcnt'] < threshold, new_columns] = np.nan
df.drop([group_col+'-'+col+'-mean' for col in agg_cols], axis=1, inplace=True)
 gc.collect()
return df
group_col = 'region_zip'
agg_cols = ['lot_sqft', 'year_built', 'finished_area_sqft_calc',
'tax_structure', 'tax_land', 'tax_property', 'property_tax_per_sqft']
prop_2016 = add_aggregate_features(prop_2016, group_col, agg_cols)
 prop_2017 = add_aggregate_features(prop_2017, group_col, agg_cols)
 prop_2017.head(10 )
epochs = 10
batch_size = 86<train_model>
train_2016 = train_2016.merge(how='left', right=prop_2016, on='parcelid')
 train_2017 = train_2017.merge(how='left', right=prop_2017, on='parcelid')
 train = pd.concat([train_2016, train_2017], axis=0, ignore_index=True)
 train = pd.concat([train_2016, train_2017], axis=0, ignore_index=True)
 print("
Combined training set size: {}".format(len(train)))
 add_simple_datetime_features(train)
 train.head(10)
<init_hyperparams>
def save_models(models):
for i, model in enumerate(models):
model.save_model('checkpoints/lgb_' + str(i))
print("Saved {} LightGBM models to files.".format(len(models)))
 def load_models(paths):
models = []
for path in paths:
model = lgb.Booster(model_file=path)
 models.append(model)
 return models
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
def lgb_drop_features(features):
unused_feature_list = ['parcelid', 'logerror']
missing_list = ['framing_id', 'architecture_style_id', 'story_id', 'perimeter_area', 'basement_sqft', 'storage_sqft']
unused_feature_list += missing_list
bad_feature_list = ['fireplace_flag', 'deck_id', 'pool_unk_1', 'construction_id', 'county_id', 'fips']
unused_feature_list += bad_feature_list
unused_feature_list += ['county_landuse_code_id', 'zoning_description_id']
return features.drop(unused_feature_list, axis=1, errors='ignore' )
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
%%time
features_2016 = prop_2016
features_2017 = prop_2017
train = train
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
lgb_features = lgb_drop_features(train)
 print("Number of features for LightGBM: {}".format(len(lgb_features.columns)))
 lgb_features.head(5)
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<set_options>
lgb_label = train.logerror.astype(np.float32)
 print(lgb_label.head())
 lgb_X = lgb_features.values
lgb_y = lgb_label.values
np.random.seed(42)
 random.seed(10)
 X_train, X_val, y_train, y_val = train_test_split(lgb_X, lgb_y, test_size=0.2)
 outlier_threshold = 0.4
mask =(abs(y_train)<= outlier_threshold)
 X_train = X_train[mask, :]
y_train = y_train[mask]
print("X_train shape: {}".format(X_train.shape))
print("y_train shape: {}".format(y_train.shape))
print("X_val shape: {}".format(X_val.shape))
print("y_val shape: {}".format(y_val.shape))
plt.style.use('ggplot')
 font = {'family' : 'meiryo'}
plt.rc('font', **font )<load_from_csv>
feature_names = [s for s in lgb_features.columns]
categorical_features = ['cooling_id', 'heating_id', 'landuse_type_id', 'year', 'month', 'quarter']
categorical_indices = []
for i, n in enumerate(lgb_features.columns):
if n in categorical_features:
categorical_indices.append(i)
 print(categorical_indices )
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv" )<correct_missing_values>
params = {}
params['objective'] = 'regression'
params['metric'] = 'mae'
params['num_threads'] = 4
params['boosting_type'] = 'gbdt'
params['num_boost_round'] = 2000
params['learning_rate'] = 0.003
params['early_stopping_rounds'] = 30
params['num_leaves'] = 127
params['min_data'] = 150
params['min_hessian'] = 0.001
params['max_depth'] = -1
params['max_bin'] = 255
params['sub_feature'] = 0.5
params['bagging_fraction'] = 0.7
params['bagging_freq'] = 50
params['min_data_per_group'] = 100
params['cat_smooth'] = 15.0
params['lambda_l1'] = 0.0
params['lambda_l2'] = 0.0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
train_df = train_df[train_df["Date"] < "2020-03-27"]
train_df = train_df.fillna("No State" )<normalization>
lgb_train_set = lgb.Dataset(X_train, label=y_train, feature_name=feature_names)
 lgb_valid_set = lgb.Dataset(X_val, label=y_val, feature_name=feature_names)
 np.random.seed(42)
 random.seed(36)
 model = lgb.train(params, lgb_train_set, verbose_eval=False,
valid_sets=[lgb_train_set, lgb_valid_set], valid_names=['train', 'val'],
categorical_feature=categorical_indices)
 print("Train score: {}".format(abs(model.predict(X_train)- y_train ).mean() * 100))
print("Val score: {}".format(abs(model.predict(X_val)- y_val ).mean() * 100))
test_rate = 0.05
maxlen = 13
train_date_count = len(set(train_df["Date"]))
X, Y = [],[]
scaler = StandardScaler()
train_df["ConfirmedCases_std"] = scaler.fit_transform(train_df["ConfirmedCases"].values.reshape(len(train_df["ConfirmedCases"].values),1))
confirmedCase_std_min = train_df["ConfirmedCases_std"].min()
train_df["ConfirmedCases_std"] = train_df["ConfirmedCases_std"] + abs(confirmedCase_std_min)
 for state,country in train_df.groupby(["Province_State","Country_Region"] ).sum().index:
df = train_df[(train_df["Country_Region"] == country)&(train_df["Province_State"] == state)]
if df["ConfirmedCases"].sum() != 0:
for i in range(len(df)- maxlen):
X.append(df[['ConfirmedCases_std']].iloc[i:(i+maxlen)].values)
 Y.append(df[['ConfirmedCases_std']].iloc[i+maxlen].values)
 X=np.array(X)
 Y=np.array(Y)
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=test_rate, shuffle = True ,random_state = 0 )<compute_test_metric>
del params['early_stopping_rounds']
params['num_boost_round'] = 1250
print(params)
 outlier_threshold = 0.4
mask =(abs(lgb_y)<= outlier_threshold)
 lgb_X = lgb_X[mask, :]
lgb_y = lgb_y[mask]
lgb_train_set = lgb.Dataset(lgb_X, label=lgb_y, feature_name=feature_names)
 print("lgb_X: {}".format(lgb_X.shape))
print("lgb_y: {}".format(lgb_y.shape))
np.random.seed(42)
 random.seed(36)
 model = lgb.train(params, lgb_train_set, verbose_eval=True, categorical_feature=categorical_indices)
 print("score: {}".format(abs(model.predict(X_val)- y_val ).mean() * 100))
def huber_loss(y_true, y_pred, clip_delta=1.0):
error = y_true - y_pred
cond = tf.keras.backend.abs(error)< clip_delta
squared_loss = 0.5 * tf.keras.backend.square(error)
 linear_loss = clip_delta *(tf.keras.backend.abs(error)- 0.5 * clip_delta)
 return tf.where(cond, squared_loss, linear_loss)
 def huber_loss_mean(y_true, y_pred, clip_delta=1.0):
return tf.keras.backend.mean(huber_loss(y_true, y_pred, clip_delta))
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5<choose_model_class>

 def transform_test_features(features_2016, features_2017):
test_features_2016 = lgb_drop_features(features_2016)
 test_features_2017 = lgb_drop_features(features_2017)
 test_features_2016['year'] = 0
test_features_2017['year'] = 1
test_features_2016['month'] = 10
test_features_2017['month'] = 10
test_features_2016['quarter'] = 4
test_features_2017['quarter'] = 4
return test_features_2016, test_features_2017

 def predict_and_export(models, features_2016, features_2017, file_name):
submission_2016 = pd.DataFrame()
submission_2017 = pd.DataFrame()
submission_2016['ParcelId'] = features_2016.parcelid
submission_2017['ParcelId'] = features_2017.parcelid
test_features_2016, test_features_2017 = transform_test_features(features_2016, features_2017)
 pred_2016, pred_2017 = [], []
for i, model in enumerate(models):
print("Start model {}(2016)".format(i))
pred_2016.append(model.predict(test_features_2016))
print("Start model {}(2017)".format(i))
pred_2017.append(model.predict(test_features_2017))
mean_pred_2016 = np.mean(pred_2016, axis=0)
 mean_pred_2017 = np.mean(pred_2017, axis=0)
 submission_2016['201610'] = [float(format(x, '.4f')) for x in mean_pred_2016]
submission_2016['201611'] = submission_2016['201610']
submission_2016['201612'] = submission_2016['201610']
submission_2017['201710'] = [float(format(x, '.4f')) for x in mean_pred_2017]
submission_2017['201711'] = submission_2017['201710']
submission_2017['201712'] = submission_2017['201710']
submission = submission_2016.merge(how='inner', right=submission_2017, on='ParcelId')
 print("Length of submission DataFrame: {}".format(len(submission)))
 print("Submission header:")
 print(submission.head())
 submission.to_csv(file_name, index=False)
 return submission, pred_2016, pred_2017
<train_model><EOS> 
file_name = '/kaggle/working/final_lgb_single.csv'
submission, pred_2016, pred_2017 = predict_and_export([model], features_2016, features_2017, file_name )