Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
<SOS> metric: MulticlassLoss Kaggle data source: two-sigma-connect:-rental-listing-inquiries<set_options>
%matplotlib inline
set_matplotlib_formats('svg')
 %matplotlib inline
sns.set_style("whitegrid" )
%matplotlib inline
sns.set()<feature_engineering>
train_data = pd.read_json('.. /input/train.json.zip', compression='zip')
 train_data.head()
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )<count_values>
train_data.isnull().sum()
train['Title'].value_counts()<count_values>
train_data['interest'] = np.where(train_data.interest_level=='low', 0,
np.where(train_data.interest_level=='medium', 1, 2))
test['Title'].value_counts()<categorify>
train_data.building_id.value_counts().nlargest(10 )
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )<drop_column>
train_data.created = pd.to_datetime(train_data.created, format='%Y-%m-%d %H:%M:%S' )
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )<categorify>
train_data['month'] = train_data.created.dt.month
train_data['day_of_week'] = train_data.created.dt.weekday
train_data['hour'] = train_data.created.dt.hour
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )<categorify>
print('Number of Unique Display Addresses is {}'.format(train_data.display_address.value_counts().shape[0]))
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )<feature_engineering>
train_data.display_address.value_counts().nlargest(15 )
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4<count_values>
train_data.latitude.nlargest(20 )
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))<data_type_conversions>
train_data.longitude.nlargest(20 )
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )<categorify>
train_data['photos_number'] = train_data.photos.str.len()
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )<categorify>
train_df = pd.read_json(".. /input/train.json.zip", compression='zip')
 test_df = pd.read_json(".. /input/test.json.zip", compression='zip')
 test_df["bathrooms"].loc[19671] = 1.5
test_df["bathrooms"].loc[22977] = 2.0
test_df["bathrooms"].loc[63719] = 2.0
train_df["price"] = train_df["price"].clip(upper=13000)
 train_df["logprice"] = np.log(train_df["price"])
 test_df["logprice"] = np.log(test_df["price"])
 train_df['half_bathrooms'] = train_df["bathrooms"] - train_df["bathrooms"].apply(int)
 test_df['half_bathrooms'] = test_df["bathrooms"] - test_df["bathrooms"].apply(int)
 train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df['price_per_room'] = train_df['price']/train_df['room_sum']
test_df['price_per_room'] = test_df['price']/test_df['room_sum']
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
train_df["created_weekday"] = train_df["created"].dt.weekday
test_df["created_weekday"] = test_df["created"].dt.weekday
train_df["created_week"] = train_df["created"].dt.week
test_df["created_week"] = test_df["created"].dt.week
train_df["pos"] = train_df.longitude.round(3 ).astype(str)+ '_' + train_df.latitude.round(3 ).astype(str)
 test_df["pos"] = test_df.longitude.round(3 ).astype(str)+ '_' + test_df.latitude.round(3 ).astype(str)
 vals = train_df['pos'].value_counts()
dvals = vals.to_dict()
train_df["density"] = train_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
test_df["density"] = test_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
features_to_use=["bathrooms", "bedrooms", "latitude", "longitude", "price","price_t","price_per_room", "logprice", "density", "half_bathrooms",
"num_photos", "num_features", "num_description_words","listing_id", "created_year", "created_month", "created_day", "created_hour", "created_week", "created_weekday"]
index=list(range(train_df.shape[0]))
random.shuffle(index)
 a=[np.nan]*len(train_df)
 b=[np.nan]*len(train_df)
 c=[np.nan]*len(train_df)
 for i in range(5):
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
test_index=index[int(( i*train_df.shape[0])/5):int(((i+1)*train_df.shape[0])/5)]
train_index=list(set(index ).difference(test_index))
for j in train_index:
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for j in test_index:
temp=train_df.iloc[j]
if sum(building_level[temp['manager_id']])!=0:
a[j]=building_level[temp['manager_id']][0]*1.0/sum(building_level[temp['manager_id']])
 b[j]=building_level[temp['manager_id']][1]*1.0/sum(building_level[temp['manager_id']])
 c[j]=building_level[temp['manager_id']][2]*1.0/sum(building_level[temp['manager_id']])
 train_df['manager_level_low']=a
train_df['manager_level_medium']=b
train_df['manager_level_high']=c
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high')
 categorical = ["street_address", "display_address", "manager_id", "building_id"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f)
 train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
 train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 train.head(50 )<feature_engineering>
skf = StratifiedKFold(n_splits=5)
 test_pred = np.zeros(( test_X.shape[0], 3))
for train_idx, val_idx in skf.split(train_X, train_y):
print(train_idx, val_idx)
 clf_lgb = lgb.LGBMClassifier(
 n_estimators = 1500,
max_depth = 6,
learning_rate = 0.02,
objective = 'multiclass',
eval_metric = 'multi_logloss',
num_class = 3,
n_jobs = -1 
)
 clf_lgb.fit(train_X[train_idx], train_y[train_idx],
eval_set =(train_X[val_idx], train_y[val_idx]),
eval_metric = 'multi_logloss',
early_stopping_rounds = 50,
verbose = 50 
)
 test_pred += clf_lgb.predict_proba(test_X)
 clf_xgb = xgb.XGBClassifier(
 n_estimators = 1500,
max_depth = 6,
learning_rate = 0.02,
objective = 'multi:softprob',
eval_metric = 'mlogloss',
num_class = 3,
n_jobs = -1 
)
 clf_xgb.fit(train_X[train_idx], train_y[train_idx],
eval_set = [(train_X[val_idx], train_y[val_idx])],
eval_metric = 'mlogloss',
early_stopping_rounds = 50,
verbose = 50 
)
 test_pred += clf_xgb.predict_proba(test_X)
 clf_cab = cab.CatBoostClassifier(
 iterations = 1500,
depth = 6,
learning_rate = 0.02,
loss_function = 'MultiClass',
classes_count = 3,
thread_count = -1 
)
 clf_cab.fit(train_X[train_idx], train_y[train_idx],
eval_set =(train_X[val_idx], train_y[val_idx]),
early_stopping_rounds = 50,
verbose = 50 
)
 test_pred += clf_cab.predict_proba(test_X)
 break
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 17, 'Fare'] = 0,
dataset.loc[(dataset['Fare'] > 17)&(dataset['Fare'] <= 30), 'Fare'] = 1,
dataset.loc[(dataset['Fare'] > 30)&(dataset['Fare'] <= 100), 'Fare'] = 2,
dataset.loc[ dataset['Fare'] > 100, 'Fare'] = 3<count_values>
out_df = pd.DataFrame(test_pred/3)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("cz.csv", index=False )