Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
<SOS> metric: MulticlassLoss Kaggle data source: two-sigma-connect:-rental-listing-inquiries<groupby>
%matplotlib inline
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))<categorify>
!pip install -U category_encoders
for c in ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','MasVnrType']:
all_data[c].fillna('None', inplace=True )<data_type_conversions>
train_data = pd.read_json('.. /input/two-sigma-connect-rental-listing-inquiries/train.json.zip', convert_dates=['created'])
 test_data = pd.read_json('.. /input/two-sigma-connect-rental-listing-inquiries/test.json.zip', convert_dates=['created'] )
all_data['MasVnrArea'].fillna(all_data['MasVnrArea'].mean() , inplace=True)
 all_data['HasMasVnr'] = all_data['MasVnrType'].apply(lambda x: 0 if x == 'None' else 1)
 all_data['HasMasVnr'] = all_data['HasMasVnr'].astype('object' )<groupby>
train_size = train_data.shape[0]
all_data['MSZoning']=all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x:x.fillna(x.mode() [0]))<categorify>
train_data['target'] = train_data['interest_level'].apply(lambda x: 0 if x=='low' else 1 if x=='medium' else 2)
 train_data['low'] = train_data['interest_level'].apply(lambda x: 1 if x=='low' else 0)
 train_data['medium'] = train_data['interest_level'].apply(lambda x: 1 if x=='medium' else 0)
 train_data['high'] = train_data['interest_level'].apply(lambda x: 1 if x=='high' else 0 )
for c in ['BsmtFullBath','BsmtHalfBath','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','GarageArea']:
all_data[c].fillna(0,inplace=True )<data_type_conversions>
full_data=pd.concat([train_data,test_data] )
for c in ['Utilities', 'Functional', 'Exterior1st', 'Exterior2nd', 'Electrical','KitchenQual', 'SaleType']:
all_data[c].fillna(all_data[c].mode() [0], inplace=True )<count_missing_values>
num_vars = ['bathrooms','bedrooms','latitude','longitude','price']
cat_vars = ['building_id','manager_id','display_address','street_address']
text_vars = ['description','features']
date_var = 'created'
image_var = 'photos'
id_var = 'listing_id'
all_data.isna().sum().nlargest()<count_values>
full_data['created_datetime'] = pd.to_datetime(full_data['created'], format="%Y-%m-%d %H:%M:%S")
 full_data['created_year']=full_data['created_datetime'].apply(lambda x:x.year)
 full_data['created_datetime'] = pd.to_datetime(full_data['created'], format="%Y-%m-%d %H:%M:%S")
 full_data['created_month']=full_data['created_datetime'].apply(lambda x:x.month)
 full_data['created_day']=full_data['created_datetime'].apply(lambda x:x.day)
 full_data['created_dayofweek']=full_data['created_datetime'].apply(lambda x:x.dayofweek)
 full_data['created_dayofyear']=full_data['created_datetime'].apply(lambda x:x.dayofyear)
 full_data['created_weekofyear']=full_data['created_datetime'].apply(lambda x:x.weekofyear)
 full_data['created_hour']=full_data['created_datetime'].apply(lambda x:x.hour)
 full_data['created_epoch']=full_data['created_datetime'].apply(lambda x:x.value//10**9)
 date_num_vars = ['created_month','created_dayofweek','created_dayofyear'
,'created_weekofyear','created_hour','created_epoch']
print([c for c in all_data.columns if(all_data[c].value_counts().iloc[0]>2700)&(all_data[c].dtype=='object')] )<drop_column>
full_data["geo_area_50"] = \
full_data[['latitude', 'longitude']]\ 
.apply(lambda x:(int(x[0]*50)%50)*50+(int(-x[1]*50)%50),axis=1)
 full_data["geo_area_100"] = \
full_data[['latitude', 'longitude']]\ 
.apply(lambda x:(int(x[0]*100)%100)*100+(int(-x[1]*100)%100),axis=1)
 full_data["geo_area_200"] = \
full_data[['latitude', 'longitude']]\ 
.apply(lambda x:(int(x[0]*200)%200)*200+(int(-x[1]*200)%200),axis=1)
 lat=40.705628
lon=-74.010278
full_data['distance_to_fi'] = full_data[['latitude', 'longitude']].apply(lambda x:math.sqrt(( x[0]-lat)**2+(x[1]-lon)**2), axis=1)
 lat = 40.785091
lon = -73.968285
full_data['distance_to_cp'] = full_data[['latitude', 'longitude']].apply(lambda x:math.sqrt(( x[0]-lat)**2+(x[1]-lon)**2), axis=1)
 geo_cat_vars = ['geo_area_50', 'geo_area_100', 'geo_area_200']
geo_num_vars = ['distance_to_fi', 'distance_to_cp']
all_data.drop(['Street', 'Utilities', 'LandSlope', 'Condition2', 'RoofMatl', 'Heating','Functional'], axis=1, inplace=True )<categorify>
full_data['rooms'] = full_data['bedrooms'] + full_data['bathrooms']
full_data['num_of_photos'] = full_data['photos'].apply(lambda x:len(x))
full_data['num_of_features'] = full_data['features'].apply(lambda x:len(x))
full_data['len_of_desc'] = full_data['description'].apply(lambda x:len(x))
full_data['words_of_desc'] = full_data['description'].apply(lambda x:len(re.sub('['+string.punctuation+']', '', x ).split()))
full_data['nums_of_desc'] = full_data['description']\ 
.apply(lambda x:re.sub('['+string.punctuation+']', '', x ).split())\ 
.apply(lambda x: len([s for s in x if s.isdigit() ]))
full_data['has_phone'] = full_data['description'].apply(lambda x:re.sub('['+string.punctuation+']', '', x ).split())\ 
.apply(lambda x: [s for s in x if s.isdigit() ])\ 
.apply(lambda x: len([s for s in x if len(str(s)) ==10])) \ 
.apply(lambda x: 1 if x>0 else 0)
 full_data['has_email'] = full_data['description'].apply(lambda x: 1 if '@renthop.com' in x else 0)
 full_data['building_id_is_zero'] = full_data['building_id'].apply(lambda x:1 if x=='0' else 0)
 additional_num_vars = ['rooms','num_of_photos','num_of_features','len_of_desc',
'words_of_desc','has_phone','has_email','building_id_is_zero']
mask=all_data['YearRemodAdd']==all_data['YearBuilt']
all_data['RemodAdd'] = all_data['YearRemodAdd'].mask(mask)
 all_data['RemodAdd'] = all_data['RemodAdd'].fillna(0)
 all_data['RemodAdd'] = all_data['RemodAdd'].apply(lambda x:1 if x != 0 else 0 ).astype('object' )<drop_column>
full_data['avg_word_len'] = full_data[['len_of_desc','words_of_desc']]\ 
.apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_room'] = full_data[['price','rooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_bedroom'] = full_data[['price','bedrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_bathroom'] = full_data[['price','bathrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_feature'] = full_data[['price','num_of_features']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_photo'] = full_data[['price','num_of_photos']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_per_word'] = full_data[['price','words_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['price_by_desc_len'] = full_data[['price','len_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['photos_per_room'] = full_data[['num_of_photos','rooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['photos_per_bedroom'] = full_data[['num_of_photos','bedrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['photos_per_bathroom'] = full_data[['num_of_photos','bathrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['desc_len_per_room'] = full_data[['len_of_desc','rooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['desc_len_per_bedroom'] = full_data[['len_of_desc','bedrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['desc_len_per_bathroom'] = full_data[['len_of_desc','bathrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['desc_len_per_word'] = full_data[['len_of_desc','words_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['desc_len_per_numeric'] = full_data[['len_of_desc','nums_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_per_room'] = full_data[['num_of_features','rooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_per_bedroom'] = full_data[['num_of_features','bedrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_per_bathroom'] = full_data[['num_of_features','bathrooms']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_per_photo'] = full_data[['num_of_features','num_of_photos']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_per_word'] = full_data[['num_of_features','words_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 full_data['features_by_desc_len'] = full_data[['num_of_features','len_of_desc']].apply(lambda x: x[0]/x[1] if x[1]!=0 else 0, axis=1)
 interactive_num_vars = ['avg_word_len','price_per_room','price_per_bedroom','price_per_bathroom',
'price_per_feature','price_per_photo','price_per_word','price_by_desc_len',
'photos_per_room','photos_per_bedroom','photos_per_bathroom',
'desc_len_per_room','desc_len_per_bedroom','desc_len_per_bathroom','desc_len_per_word',
'desc_len_per_numeric','features_per_room','features_per_bedroom','features_per_bathroom',
'features_per_photo','features_per_word','features_by_desc_len']
all_data.drop('YearRemodAdd', axis=1, inplace=True )<feature_engineering>
%%time
display=full_data["display_address"].value_counts()
manager_id=full_data["manager_id"].value_counts()
building_id=full_data["building_id"].value_counts()
street=full_data["street_address"].value_counts()
bedrooms=full_data["bedrooms"].value_counts()
bathrooms=full_data["bathrooms"].value_counts()
created_dayofyear=full_data["created_dayofyear"].value_counts()
created_weekofyear=full_data["created_weekofyear"].value_counts()
full_data["display_count"]=full_data["display_address"].apply(lambda x:display[x])
 full_data["manager_count"]=full_data["manager_id"].apply(lambda x:manager_id[x])
 full_data["building_count"]=full_data["building_id"].apply(lambda x:building_id[x])
 full_data["street_count"]=full_data["street_address"].apply(lambda x:street[x])
 full_data["bedrooms_count"]=full_data["bedrooms"].apply(lambda x:bedrooms[x])
 full_data["bathrooms_count"]=full_data["bathrooms"].apply(lambda x:bathrooms[x])
 full_data["created_dayofyear_count"]=full_data["created_dayofyear"].\
apply(lambda x:created_dayofyear[x])
 full_data["created_weekofyear_count"]=full_data["created_weekofyear"].\
apply(lambda x:created_weekofyear[x])
 count_vars = ['manager_count', 'building_count', 'street_count', 'bedrooms_count',
'bathrooms_count', 'created_dayofyear_count', 'created_weekofyear_count']
all_data['TotalSF']=all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['TotalBathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['TotalPorchSF'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF'] )<data_type_conversions>
num_cat_vars =[]
price_by_manager = full_data.groupby('manager_id')['price'].agg([np.min,np.max,np.median,np.mean] ).reset_index()
price_by_manager.columns = ['manager_id','min_price_by_manager',
'max_price_by_manager','median_price_by_manager','mean_price_by_manager']
full_data = pd.merge(full_data,price_by_manager, how='left',on='manager_id')
 price_by_building = full_data.groupby('building_id')['price'].agg([np.min,np.max,np.median,np.mean] ).reset_index()
price_by_building.columns = ['building_id','min_price_by_building',
'max_price_by_building','median_price_by_building','mean_price_by_building']
full_data = pd.merge(full_data,price_by_building, how='left',on='building_id')
 full_data['price_percentile_by_manager']=\
full_data[['price','min_price_by_manager','max_price_by_manager']]\ 
.apply(lambda x:(x[0]-x[1])/(x[2]-x[1])if(x[2]-x[1])!=0 else 0.5,
axis=1)
 full_data['price_percentile_by_building']=\
full_data[['price','min_price_by_building','max_price_by_building']]\ 
.apply(lambda x:(x[0]-x[1])/(x[2]-x[1])if(x[2]-x[1])!=0 else 0.5,
axis=1)
 num_cat_vars.append('price_percentile_by_manager')
 num_cat_vars.append('price_percentile_by_building')
 print(num_cat_vars )
all_data['has1stfloor'] = all_data['1stFlrSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasBsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasOpenPorch'] = all_data['OpenPorchSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['has3SsnPorch'] = all_data['3SsnPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasEnclosedPorch'] = all_data['EnclosedPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasScreenPorch'] = all_data['ScreenPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasWoodDeckPorch'] = all_data['WoodDeckSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object' )<drop_column>
for comb in itertools.combinations(cat_vars, 2):
comb_var_name = comb[0] +'-'+ comb[1]
full_data [comb_var_name] = full_data [ comb[0]].astype(str)+'_' + full_data [ comb[1]].astype(str)
 cat_vars.append(comb_var_name)
 cat_vars
all_data.drop(['TotalBsmtSF','1stFlrSF','2ndFlrSF','FullBath','HalfBath','BsmtFullBath','BsmtHalfBath','OpenPorchSF','3SsnPorch',
'EnclosedPorch','ScreenPorch','WoodDeckSF'],
axis=1,inplace=True )<drop_column>
full_data["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
cntvec = CountVectorizer(stop_words='english', max_features=200)
 feature_sparse =cntvec.fit_transform(full_data["features"]\ 
.apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x])))
 feature_vars = ['feature_' + v for v in cntvec.vocabulary_]
cntvec = CountVectorizer(stop_words='english', max_features=100)
 desc_sparse = cntvec.fit_transform(full_data["description"])
 desc_vars = ['desc_' + v for v in cntvec.vocabulary_]
cntvec = CountVectorizer(stop_words='english', max_features=10)
 st_addr_sparse = cntvec.fit_transform(full_data["street_address"])
 st_addr_vars = ['desc_' + v for v in cntvec.vocabulary_]
all_data.drop(['TotRmsAbvGrd'],axis=1,inplace=True )<drop_column>
LBL = preprocessing.LabelEncoder()
LE_vars=[]
LE_map=dict()
for cat_var in cat_vars:
print("Label Encoding %s" %(cat_var))
LE_var=cat_var+'_le'
full_data[LE_var]=LBL.fit_transform(full_data[cat_var])
 LE_vars.append(LE_var)
 LE_map[cat_var]=LBL.classes_
print("Label-encoded feaures: %s" %(LE_vars))
all_data.drop('Id',axis=1,inplace=True )<categorify>
OHE = preprocessing.OneHotEncoder(sparse=True)
 start=time.time()
OHE.fit(full_data[LE_vars])
 OHE_sparse=OHE.transform(full_data[LE_vars])
 print('One-hot-encoding finished in %f seconds' %(time.time() -start))
OHE_vars = [var[:-3] + '_' + str(level ).replace(' ','_')\
for var in cat_vars for level in LE_map[var] ]
print("OHE_sparse size :" ,OHE_sparse.shape)
 print("One-hot encoded catgorical feature samples : %s" %(OHE_vars[:100]))
all_data_dum = pd.get_dummies(all_data, drop_first=True)
 all_data_dum.shape<define_variables>
full_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars+ count_vars + LE_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )
num_col=[c for c in all_data.columns if all_data[c].dtype !='object']
print(num_col )<feature_engineering>
class MeanEncoder:
def __init__(self, categorical_features, n_splits=5, target_type='classification', prior_weight_func=None):

 self.categorical_features = categorical_features
self.n_splits = n_splits
self.learned_stats = {}
if target_type == 'classification':
self.target_type = target_type
self.target_values = []
else:
self.target_type = 'regression'
self.target_values = None
if isinstance(prior_weight_func, dict):
self.prior_weight_func = eval('lambda x: 1 /(1 + np.exp(( x - k)/ f)) ', dict(prior_weight_func, np=np))
elif callable(prior_weight_func):
self.prior_weight_func = prior_weight_func
else:
self.prior_weight_func = lambda x: 1 /(1 + np.exp(( x - 2)/ 1))
@staticmethod
def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_weight_func):
X_train = X_train[[variable]].copy()
X_test = X_test[[variable]].copy()
if target is not None:
nf_name = '{}_pred_{}'.format(variable, target)
 X_train['pred_temp'] =(y_train == target ).astype(int)
 else:
nf_name = '{}_pred'.format(variable)
 X_train['pred_temp'] = y_train
prior = X_train['pred_temp'].mean()
col_avg_y = X_train.groupby(by=variable, axis=0)['pred_temp'].agg({'mean': 'mean', 'beta': 'size'})
 col_avg_y['beta'] = prior_weight_func(col_avg_y['beta'])
 col_avg_y[nf_name] = col_avg_y['beta'] * prior +(1 - col_avg_y['beta'])* col_avg_y['mean']
col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True)
 nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values
nf_test = X_test.join(col_avg_y, on=variable ).fillna(prior, inplace=False)[nf_name].values
return nf_train, nf_test, prior, col_avg_y
def fit_transform(self, X, y):

 X_new = X.copy()
if self.target_type == 'classification':
skf = StratifiedKFold(self.n_splits)
 else:
skf = KFold(self.n_splits)
 if self.target_type == 'classification':
self.target_values = sorted(set(y))
self.learned_stats = {'{}_pred_{}'.format(variable, target): [] for variable, target in
product(self.categorical_features, self.target_values)}
for variable, target in product(self.categorical_features, self.target_values):
nf_name = '{}_pred_{}'.format(variable, target)
 X_new.loc[:, nf_name] = np.nan
for large_ind, small_ind in skf.split(y, y):
nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(
 X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, target, self.prior_weight_func)
 X_new.iloc[small_ind, -1] = nf_small
self.learned_stats[nf_name].append(( prior, col_avg_y))
else:
self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features}
for variable in self.categorical_features:
nf_name = '{}_pred'.format(variable)
 X_new.loc[:, nf_name] = np.nan
for large_ind, small_ind in skf.split(y, y):
nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(
 X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, None, self.prior_weight_func)
 X_new.iloc[small_ind, -1] = nf_small
self.learned_stats[nf_name].append(( prior, col_avg_y))
return X_new
def transform(self, X):

 X_new = X.copy()
if self.target_type == 'classification':
for variable, target in product(self.categorical_features, self.target_values):
nf_name = '{}_pred_{}'.format(variable, target)
 X_new[nf_name] = 0
for prior, col_avg_y in self.learned_stats[nf_name]:
X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable ).fillna(prior, inplace=False)[
nf_name]
X_new[nf_name] /= self.n_splits
else:
for variable in self.categorical_features:
nf_name = '{}_pred'.format(variable)
 X_new[nf_name] = 0
for prior, col_avg_y in self.learned_stats[nf_name]:
X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable ).fillna(prior, inplace=False)[
nf_name]
X_new[nf_name] /= self.n_splits
return X_new
log=PowerTransformer()
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(X[c].skew())
 log.fit(X[c].values.reshape(-1,1))
skew_after.append(skew(( log.transform(X[c].values.reshape(-1,1)))) [0])
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df = skew_df[skew_df['skew_diff']>0.1]
columns_to_work_log = list(skew_df.index)
 skew_df<categorify>
mean_encoder = MeanEncoder(categorical_features=['manager_id','building_id'])
 mean_encoded_train = mean_encoder.fit_transform(train_data, train_data['target'])
 mean_encoded_test = mean_encoder.transform(test_data)
 mean_coded_vars = list(set(mean_encoded_train.columns)- set(train_data.columns))
mean_coded_vars.append('listing_id')
 full_data = pd.merge(full_data,
pd.concat([mean_encoded_train[mean_coded_vars], mean_encoded_test[mean_coded_vars]]),
how='left',
on='listing_id'
)
logx = PowerTransformer()
for c in ['LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'LowQualFinSF', 'GrLivArea', 'KitchenAbvGr', 'Fireplaces', 'PoolArea', 'MoSold', 'TotalSF', 'TotalBathrooms', 'TotalPorchSF']:
X[c] = logx.fit_transform(X[[c]])
 X_test[c] = logx.transform(X_test[[c]] )<feature_engineering>
CB_encoder = PolynomialWrapper(CatBoostEncoder())
 train_cb = CB_encoder.fit_transform(full_data[:train_size][cat_vars], full_data[:train_size]['target'])
 test_cb = CB_encoder.transform(full_data[train_size:][cat_vars])
 CB_vars = [f'cb_{c}' for c in train_cb.columns]
train_cb.columns = CB_vars
test_cb.columns = CB_vars
full_data = pd.concat([full_data, pd.concat([train_cb, test_cb])], axis=1 )
logy = PowerTransformer()
y = logy.fit_transform(y.values.reshape(-1,1))<import_modules>
full_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars+ count_vars + LE_vars + mean_coded_vars + CB_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )
from xgboost import XGBRegressor
from sklearn.linear_model import ElasticNet, Lasso, LassoLarsCV
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.model_selection import cross_val_score, RandomizedSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.feature_selection import RFE
from sklearn.decomposition import PCA<compute_train_metric>
min_listing_id = full_data['listing_id'].min()
max_listing_id = full_data['listing_id'].max()
full_data['norm_listing_id']=full_data['listing_id'].apply(lambda x:np.float64(( x-min_listing_id+1)) /(max_listing_id-min_listing_id+1))
listing_vars = [ 'norm_listing_id']
def rmse_cv(model):
neg_mse = cross_val_score(model, X,y, cv=5, scoring='neg_mean_squared_error')
 rmse=np.sqrt(-neg_mse)
 print('RMSE mean: {0:.5f} , RMSE std: {1:.5f}'.format(rmse.mean() , rmse.std()))<compute_train_metric>
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars + count_vars \
+ listing_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )
def rmse_cv_return(model):
neg_mse = cross_val_score(model, X,y, cv=5, scoring='neg_mean_squared_error')
 rmse=np.sqrt(-neg_mse)
 return rmse.mean()<choose_model_class>
mkt_price = full_data.groupby(['building_id', 'display_address', 'bedrooms', 'bathrooms'] ).price.mean().reset_index()
mkt_price = pd.merge(full_data[['building_id', 'display_address', 'bedrooms', 'bathrooms']],
mkt_price, how='left', on=['building_id', 'display_address', 'bedrooms', 'bathrooms'] ).price
full_data['mkt_price'] = mkt_price.values
full_data['diff_to_mkt_price'] = full_data['price'] - full_data['mkt_price']
full_data['ratio_to_mkt_price'] = full_data['price'] / full_data['mkt_price']
price_vars = ['diff_to_mkt_price', 'ratio_to_mkt_price']
model_xgb = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor', random_state=1111)
 rmse_cv(model_xgb )<define_search_model>
full_data['photos_str'] = full_data['photos'].astype(str)
 full_data['listing_uid'] = full_data[['manager_id', 'building_id','photos_str']].apply(lambda x: hashlib.md5(( x[0] + x[1] + x[2] ).encode() ).hexdigest() , axis=1)
 full_data['posted_times'] = full_data.groupby('listing_uid' ).created_datetime.rank(method='first', na_option='top',pct=True)
 full_data['num_of_html_tag']=full_data.description.apply(lambda x:x.count('<'))
full_data['num_of_
full_data['num_of_!']=full_data.description.apply(lambda x:x.count('!'))
full_data['num_of_$']=full_data.description.apply(lambda x:x.count('$'))
full_data['num_of_*']=full_data.description.apply(lambda x:x.count('*'))
full_data['num_of_>']=full_data.description.apply(lambda x:x.count('>'))
full_data['num_of_puncs']=full_data['num_of_
full_data['puncs_ratio'] = full_data['num_of_puncs']/full_data['len_of_desc']
full_data['upper_char_ratio'] = full_data['description'].apply(lambda x: 0 if sum([s.isalpha() for s in x])==0 else sum([s.isalpha() &s.isupper() for s in x])/ sum([s.isalpha() for s in x]))
full_data['disp_is_street'] =(full_data['display_address'] == full_data['street_address'])*1
full_data['disp_st_addr_word_ratio'] = full_data.apply(lambda x:len(x['display_address'].split(' ')) /len(x['street_address'].split(' ')) , axis=1)
 listing_quality_vars = ['disp_is_street', 'num_of_html_tag','num_of_
'posted_times', 'disp_st_addr_word_ratio','upper_char_ratio']
model_xgboost1 = make_pipeline(StandardScaler() , XGBRegressor())
 rmse_cv(model_xgboost1 )<choose_model_class>
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars + count_vars \
+ listing_vars + listing_quality_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )
model_xgb_tuned = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=4.31172,n_estimators=800,max_depth=2,learning_rate=0.112414 ,random_state=1111)
 rmse_cv(model_xgb_tuned )<choose_model_class>
def p25(x):
return np.percentile(x, 25)
 def p50(x):
return np.percentile(x, 50)
 def p75(x):
return np.percentile(x, 75)
 def nunique(x):
return np.size(np.unique(x))
def max_min(x):
return np.max(x)-np.min(x)
 def p75_p25(x):
return np.percentile(x, 75)-np.percentile(x, 25)
 def get_group_stats(df, stat_funcs, target_column, group_column, ranking=False, ranking_pct=True):
aggr = df.groupby(group_column)[target_column].agg([v for v in stat_funcs.values() ] ).reset_index()
aggr.columns = [group_column] + [ target_column + '_' + k + '_by_' + group_column for k in stat_funcs.keys() ]
aggr = df[[group_column]].merge(aggr, how='left', on=group_column)
 if ranking:
aggr[target_column + '_rank_by_' + group_column] = df.groupby(group_column)[target_column].rank(method='dense',
na_option='top',
pct=ranking_pct)
 return aggr.drop(group_column, axis=1 )
model_xgb_tuned1 = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=17.144285714285715,n_estimators=500,max_depth=3,learning_rate=0.07827586206896552 ,random_state=1111)
 rmse_cv(model_xgb_tuned1 )<choose_model_class>
stat_funcs = {
'mean': np.mean,
'min': np.min,
'max': np.max,
'std': np.std,
'p25': p25,
'p50': p50,
'p75': p75,
'skew': skew,
'kurtosis': kurtosis,
'max_min': max_min,
'p75_p25': p75_p25
}
mgr_aggr = pd.DataFrame()
for num_var in num_vars + additional_num_vars + listing_quality_vars:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, stat_funcs,
target_column=num_var, group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'max_min': max_min, 'p75_p25': p75_p25},
target_column='created_epoch', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='created_dayofyear', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='building_id', group_column='manager_id', ranking=False)
 ],
axis=1 
)
 for aggr_col in ['geo_area_50', 'geo_area_100', 'geo_area_200']:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column=aggr_col, group_column='manager_id', ranking=False)
 ],
axis=1 
)
 for aggr_col in ['diff_to_mkt_price', 'ratio_to_mkt_price']:
mgr_aggr = pd.concat([mgr_aggr,
get_group_stats(full_data, {'mean': np.mean},
target_column=aggr_col, group_column='manager_id', ranking=False)
 ],
axis=1
)
model_xgb_tuned1_sc = make_pipeline(StandardScaler() ,XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=17.144285714285715,n_estimators=500,max_depth=3,learning_rate=0.07827586206896552 ,random_state=1111))
rmse_cv(model_xgb_tuned1_sc )<choose_model_class>
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars + count_vars \
+ listing_vars + listing_quality_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,
"testing data size: ", test_x.shape )
model_xgb_tuned2 = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=16.362109350140535,n_estimators=400,max_depth=2,learning_rate=0.04200863893086535 ,random_state=1111)
 rmse_cv(model_xgb_tuned2 )<choose_model_class>
stat_funcs = {
'mean': np.mean,
'min': np.min,
'max': np.max,
'std': np.std,
'p25': p25,
'p50': p50,
'p75': p75,
'skew': skew,
'kurtosis': kurtosis,
'max_min': max_min,
'p75_p25': p75_p25
}
building_aggr = pd.DataFrame()
building_aggr = pd.concat([building_aggr,
get_group_stats(full_data, stat_funcs,
target_column='price', group_column='building_id', ranking=False)
 ],
axis=1 
)
 building_aggr = pd.concat([building_aggr,
get_group_stats(full_data, {'nunique': nunique},
target_column='manager_id', group_column='building_id', ranking=False)
 ],
axis=1
)
model_xgb_tuned2_sc = make_pipeline(StandardScaler() ,XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=16.362109350140535,n_estimators=400,max_depth=2,learning_rate=0.04200863893086535 ,random_state=1111))
rmse_cv(model_xgb_tuned2_sc )<compute_train_metric>
park_listings = full_data[full_data[['description', 'features']].apply(lambda x: 'park' in x[0] or 'park' in x[1], axis=1)][['latitude', 'longitude']]
park_n_clusters = 25
kms = KMeans(n_clusters=park_n_clusters)
 kms.fit(park_listings)
 park_dist_data = pd.DataFrame(kms.transform(full_data[['latitude', 'longitude']]),
columns = ['dist_to_park_' + str(i)for i in range(park_n_clusters)]
)
<predict_on_test>
subway_listings = full_data[full_data[['description', 'features']].apply(lambda x: 'subway' in x[0] or 'subway' in x[1], axis=1)][['latitude', 'longitude']]
subway_n_clusters = 400
kms = KMeans(n_clusters=subway_n_clusters)
 kms.fit(subway_listings)
 subway_dist_data = pd.DataFrame(kms.transform(full_data[['latitude', 'longitude']]),
columns = ['dist_to_subway_' + str(i)for i in range(subway_n_clusters)]
)
model_xgb_tuned2.fit(X,y)
 y_pred_test = model_xgb_tuned2.predict(X_test)
 y_pred_test = logy.inverse_transform(y_pred_test.reshape(-1,1))
y_pred_test = y_pred_test.reshape(-1 )<save_to_csv>
nlp = spacy.load("en_core_web_sm")
 def seq_to_token(seq, nlp=nlp):
doc = nlp(str(seq ).lower())
 tokens = [token.text for token in doc if not(token.is_space | token.is_stop|token.like_num)]
return tokens
def tokens_to_vec(tokens, model, vec_size=10):
if len(tokens)==0:
return np.zeors(vec_size)
 else:
return np.array([emb_model[token] for token in tokens] ).mean(axis=1 )
output = pd.DataFrame({'Id': test['Id'],
'SalePrice': y_pred_test})
 output.to_csv('submission.csv', index=False )<import_modules>
building_model = FastText(size=10, window=3, min_count=1, workers=16)
 building_model.build_vocab(sentences=building_by_mgr)
 building_model.train(sentences=building_by_mgr.values, total_examples=len(building_by_mgr.values), epochs=5)
 building_model['8a8b08e08888819a3e745005a8cd0408']
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.utils import np_utils<train_model>
building_emb = full_data['building_id'].apply(lambda x:building_model[x] ).values
building_emb = np.array([e.reshape(1,-1)for e in building_emb] ).reshape(-1,10)
 building_emb
( X_train, y_train),(X_test, y_test)= mnist.load_data()<data_type_conversions>
manager_by_building = full_data.groupby('building_id')['manager_id'].apply(list)
 manager_model = FastText(size=10, window=3, min_count=1, workers=16)
 manager_model.build_vocab(sentences=manager_by_building)
 manager_model.train(sentences=manager_by_building.values,
total_examples=len(manager_by_building.values), epochs=5)
 manager_emb = full_data['manager_id'].apply(lambda x:manager_model[x] ).values
manager_emb = np.array([e.reshape(1,-1)for e in manager_emb] ).reshape(-1,10)
 manager_emb
num_pixels = X_train.shape[1] * X_train.shape[2]
X_train = X_train.reshape(( X_train.shape[0], num_pixels)).astype('float32')
 X_test = X_test.reshape(( X_test.shape[0], num_pixels)).astype('float32' )<feature_engineering>
manager_by_building = full_data.groupby('building_id')['manager_id'].apply(list)
 manager_model = FastText(size=10, window=3, min_count=1, workers=16)
 manager_model.build_vocab(sentences=manager_by_building)
 manager_model.train(sentences=manager_by_building.values,
total_examples=len(manager_by_building.values), epochs=5)
 manager_emb = full_data['manager_id'].apply(lambda x:manager_model[x] ).values
manager_emb = np.array([e.reshape(1,-1)for e in manager_emb] ).reshape(-1,10)
 manager_emb
X_train = X_train / 255
X_test = X_test / 255<categorify>
desc_vec = np.vstack(( pickle.load(open('/kaggle/input/text-featurizer/train_desc_vec_lg.pkl', 'rb')) ,
pickle.load(open('/kaggle/input/text-featurizer/test_desc_vec_lg.pkl', 'rb'))))
y_train = np_utils.to_categorical(y_train)
 y_test = np_utils.to_categorical(y_test)
 num_classes = y_test.shape[1]<choose_model_class>
image_date = pd.read_csv(".. /input/twosigma-magic-feature/listing_image_time.csv")
 image_date.columns = ["listing_id", "image_time_stamp"]
full_data = pd.merge(full_data, image_date, on="listing_id", how="left")
 magic_vars = ['image_time_stamp']
def baseline_model() :
	
	model = Sequential()
	model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
	model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
	
	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 	return model<train_model>
full_num_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+geo_num_vars+ geo_cat_vars + count_vars \
+ listing_vars + listing_quality_vars + magic_vars + price_vars
full_cat_vars = LE_vars + mean_coded_vars + CB_vars
full_vars = full_num_vars + full_cat_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr,
building_aggr,
park_dist_data,
subway_dist_data,
manager_emb,
building_emb,
desc_vec] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse,
mgr_aggr,
building_aggr,
park_dist_data,
subway_dist_data,
manager_emb,
building_emb,
desc_vec] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
print("training data size: ", train_x.shape,
"testing data size: ", test_x.shape )
<data_type_conversions><EOS> 
sparse.save_npz('train_x.npz', train_x)
 sparse.save_npz('test_x.npz', test_x)