Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
<SOS> metric: MulticlassLoss Kaggle data source: two-sigma-connect:-rental-listing-inquiries<count_values>
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import random
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from collections import defaultdict, Counter
freq = train['Embarked'].value_counts().index[0]
train['Embarked'].fillna(freq, inplace=True )<drop_column>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model
train.drop(['Ticket', 'Cabin'], axis=1, inplace=True)
 train<categorify>
data_path = ".. /input/two-sigma-connect-rental-listing-inquiries/"
train_file = data_path + "train.json.zip"
test_file = data_path + "test.json.zip"
train_df = pd.read_json(train_file, compression='zip')
 test_df = pd.read_json(test_file, compression='zip')
 features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]
train = pd.get_dummies(train, columns=['Embarked', 'Sex'], prefix=['Embarked', 'Sex'], prefix_sep=' - ')
 train<count_values>
mean_price = int(train_df['price'].mean())
 test_df.loc[test_df['price']<200,'price'] = mean_price
train_df.loc[train_df['price']<200,'price'] = mean_price
list_handling = train['Name'].apply(lambda x: x.split(',')[1].split('.')[0] ).value_counts()
list_handling<feature_engineering>
test_df['features'] = test_df['features'].apply(lambda x: list(map(str.strip, map(str.lower, x)))) \ 
.apply(lambda x: [dict_rep_features[i] for i in x if i in dict_rep_features.index])\ 
.apply(lambda x: list(set(x)))
 train_df['features'] = train_df['features'].apply(lambda x: list(map(str.strip, map(str.lower, x)))) \ 
.apply(lambda x: [dict_rep_features[i] for i in x if i in dict_rep_features.index])\ 
.apply(lambda x: list(set(x)))
for elem in list_handling.index:
cond1 = train['Name'].str.contains(elem + '.', regex=False)
 cond2 = train['Age'].isnull()
train.loc[cond1 & cond2, 'Age'] = train.loc[cond1, 'Age'].mean()<count_missing_values>
def cart2rho(x, y):
rho = np.sqrt(x**2 + y**2)
 return rho
def cart2phi(x, y):
phi = np.arctan2(y, x)
 return phi
def rotation_x(row, alpha):
x = row['latitude']
y = row['longitude']
return x*math.cos(alpha)+ y*math.sin(alpha)
 def rotation_y(row, alpha):
x = row['latitude']
y = row['longitude']
return y*math.cos(alpha)- x*math.sin(alpha)
 def add_rotation(degrees, df):
namex = "rot" + str(degrees)+ "_X"
namey = "rot" + str(degrees)+ "_Y"
df['num_' + namex] = df.apply(lambda row: rotation_x(row, math.pi/(180/degrees)) , axis=1)
 df['num_' + namey] = df.apply(lambda row: rotation_y(row, math.pi/(180/degrees)) , axis=1)
 return df
def operate_on_coordinates(tr_df, te_df):
for df in [tr_df, te_df]:
df["num_rho"] = df.apply(lambda x: cart2rho(x["latitude"] - 40.78222222, x["longitude"]+73.96527777), axis=1)
 df["num_phi"] = df.apply(lambda x: cart2phi(x["latitude"] - 40.78222222, x["longitude"]+73.96527777), axis=1)
 for angle in [15,30,45,60]:
df = add_rotation(angle, df)
 return tr_df, te_df
train_df, test_df = operate_on_coordinates(train_df, test_df)
 features_to_use.extend(['num_rho', 'num_phi', 'num_rot15_X', 'num_rot15_Y', 'num_rot30_X',
'num_rot30_Y', 'num_rot45_X', 'num_rot45_Y', 'num_rot60_X',
'num_rot60_Y'] )
nans = train.isnull().sum()
nans[nans != 0]<count_missing_values>
def cap_share(x):
return sum(1 for c in x if c.isupper())/float(len(x)+1)
 for df in [train_df, test_df]:
df['num_cap_share'] = df['description'].apply(cap_share)
 df['num_nr_of_lines'] = df['description'].apply(lambda x: x.count('<br /><br />'))
df['num_redacted'] = 0
df['num_redacted'].ix[df['description'].str.contains('website_redacted')] = 1
df['num_email'] = 0
df['num_email'].ix[df['description'].str.contains('@')] = 1
reg = re.compile(".*?(\(?\d{3}\D{0,3}\d{3}\D{0,3}\d{4} ).*?", re.S)
 def try_and_find_nr(description):
if reg.match(description)is None:
return 0
return 1
df['num_phone_nr'] = df['description'].apply(try_and_find_nr)
 features_to_use.extend(['num_cap_share', 'num_nr_of_lines', 'num_redacted',
'num_email', 'num_phone_nr'] )
test.isnull().sum()<count_missing_values>
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"] )
test.isnull().sum()<categorify>
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
features_to_use.extend(["price_t", "room_sum", "num_description_words"] )
test = pd.get_dummies(test, columns=['Embarked', 'Sex'], prefix=['Embarked', 'Sex'], prefix_sep=' - ')
 test.drop(['Cabin', 'Ticket'], axis=1, inplace=True)
 test<feature_engineering>
start_values = [0,0,0]
index=list(range(train_df.shape[0]))
random.shuffle(index)
 a=[np.nan]*len(train_df)
 b=[np.nan]*len(train_df)
 c=[np.nan]*len(train_df)
 for i in range(5):
building_level={}
for j in train_df['manager_id'].values:
building_level[j]= start_values.copy()
test_index=index[int(( i*train_df.shape[0])/5):int(((i+1)*train_df.shape[0])/5)]
train_index=list(set(index ).difference(test_index))
for j in train_index:
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for j in test_index:
temp=train_df.iloc[j]
if sum(building_level[temp['manager_id']])!=0:
a[j]=building_level[temp['manager_id']][0]*1.0/sum(building_level[temp['manager_id']])
 b[j]=building_level[temp['manager_id']][1]*1.0/sum(building_level[temp['manager_id']])
 c[j]=building_level[temp['manager_id']][2]*1.0/sum(building_level[temp['manager_id']])
 train_df['manager_level_low']=a
train_df['manager_level_medium']=b
train_df['manager_level_high']=c
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]= start_values.copy()
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high' )
for elem in list_handling.index:
cond1 = test['Name'].str.contains(elem + '.', regex=False)
 cond2 = test['Age'].isnull()
test.loc[cond1 & cond2, 'Age'] = train.loc[train['Name'].str.contains(elem + '.', regex=False), 'Age'].mean()
cond3 = test['Fare'].isnull()
test.loc[cond3, 'Fare'] = train.loc[train['Name'].str.contains(elem + '.', regex=False), 'Fare'].mean()<drop_column>
train_df["listing_id1"] = train_df["listing_id"] - 68119576.0
test_df["listing_id1"] = test_df["listing_id"] - 68119576.0
train_df["num_price_by_furniture"] =(train_df["price"])/(train_df["bathrooms"] + train_df["bedrooms"] + 1.0)
 test_df["num_price_by_furniture"] =(test_df["price"])/(test_df["bathrooms"] + test_df["bedrooms"] + 1.0)
 train_df["price_latitue"] =(train_df["price"])/(train_df["latitude"]+1.0)
 test_df["price_latitue"] =(test_df["price"])/(test_df["latitude"]+1.0)
 train_df["price_longtitude"] =(train_df["price"])/(train_df["longitude"]-1.0)
 test_df["price_longtitude"] =(test_df["price"])/(test_df["longitude"]-1.0)
 train_df["num_furniture"] = train_df["bathrooms"] + train_df["bedrooms"]
test_df["num_furniture"] = test_df["bathrooms"] + test_df["bedrooms"]
train_df["total_days"] =(train_df["created_month"] -4.0)*30 + train_df["created_day"] + train_df["created_hour"] /25.0
test_df["total_days"] =(test_df["created_month"] -4.0)*30 + test_df["created_day"] + test_df["created_hour"] /25.0
train_df["diff_rank"]= train_df["total_days"]/train_df["listing_id1"]
test_df["diff_rank"]= test_df["total_days"]/test_df["listing_id1"]
features_to_use.extend([ "total_days","diff_rank",
"num_price_by_furniture","price_latitue","price_longtitude",'num_furniture'] )
train.drop(['Name'], axis=1, inplace=True)
 test.drop(['Name'], axis=1, inplace=True )<prepare_x_and_y>
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )
Y_train = train['Survived'].values
X_train = train.drop(['Survived'], axis=1 ).values
X_test = test.values<import_modules>
train_df["price0"] =(train_df["price"]%10==0 ).astype(int)
 test_df["price0"] =(test_df["price"]%10==0 ).astype(int)
 train_df["manager_count"] = train_df["manager_id"].replace(train_df["manager_id"].value_counts())
 test_df["manager_count"] = test_df["manager_id"].replace(train_df["manager_id"].value_counts())
 features_to_use.extend(["price0",'manager_count'] )
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout<feature_engineering>
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=70)
 te_sparse = tfidf.fit_transform(test_df["features"])
 tr_sparse = tfidf.transform(train_df["features"] )
mean = X_train.mean(axis=0)
 std = X_train.std(axis=0)
 X_train -= mean
X_train /= std
X_test -= mean
X_test /= std<choose_model_class>
tfidfdesc=TfidfVectorizer(min_df=20, max_features=50, strip_accents='unicode',lowercase =True,
analyzer='word', token_pattern=r'\w{16,}', ngram_range=(1, 2), use_idf=False,smooth_idf=False,
sublinear_tf=True, stop_words = 'english')
 train_df['description'] = train_df['description'].apply(lambda x: str(x ).encode('utf-8')if len(x)>2 else "nulldesc")
 test_df['description'] = test_df['description'].apply(lambda x: str(x ).encode('utf-8')if len(x)>2 else "nulldesc")
 te_sparsed = tfidfdesc.fit_transform(test_df["description"])
 tr_sparsed = tfidfdesc.transform(train_df["description"] )
model = Sequential()
model.add(Dense(40, kernel_initializer='uniform', activation='relu', input_shape=(X_train.shape[1],)))
 model.add(Dropout(0.5))
model.add(Dense(100, kernel_initializer='uniform', activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(40, kernel_initializer='uniform', activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))
model.compile(optimizer=optimizers.Adam(lr=1e-5), loss='binary_crossentropy', metrics=['acc'])
 model.summary()<train_model>
train_X = sparse.hstack([train_df[features_to_use], tr_sparse,tr_sparsed] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse,te_sparsed] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )
checkpoint_path = 'bestmodel4.hdf5'
checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
 callbacks_list = [checkpoint]
history = model.fit(X_train, Y_train, batch_size=30, epochs=3000,
callbacks=callbacks_list, verbose=0, validation_split=0.2 )<predict_on_test>
preds, model = runXGB(train_X, train_y, test_X, num_rounds=1000)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )