Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
train.Cabin.value_counts()<feature_engineering>
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json')
 train_df.head()
test_df["bathrooms"].loc[19671] = 1.5
test_df["bathrooms"].loc[22977] = 2.0
test_df["bathrooms"].loc[63719] = 2.0
ulimit = np.percentile(train_df.price.values, 99)
 train_df['price'].ix[train_df['price']>ulimit] = ulimit
ulimit = np.percentile(test_df.price.values, 99)
 test_df['price'].ix[test_df['price']>ulimit] = ulimit
train_df["logprice"] = np.log(train_df["price"])
 test_df["logprice"] = np.log(test_df["price"])
 train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["pos"] = train_df.longitude.round(3 ).astype(str)+ '_' + train_df.latitude.round(3 ).astype(str)
 test_df["pos"] = test_df.longitude.round(3 ).astype(str)+ '_' + test_df.latitude.round(3 ).astype(str)
 vals = train_df['pos'].value_counts()
dvals = vals.to_dict()
train_df["density"] = train_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
test_df["density"] = test_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df['price_per_room'] = train_df['price']/train_df['room_sum']
test_df['price_per_room'] = test_df['price']/test_df['room_sum']
image_date = pd.read_csv('.. /input/listing_image_time.csv')
 image_date.columns = ["listing_id", "time_stamp"]
image_date.loc[80240,"time_stamp"] = 1478129766
image_date["img_date"] = pd.to_datetime(image_date["time_stamp"], unit="s")
 image_date["img_days_passed"] =(image_date["img_date"].max() - image_date["img_date"] ).astype("timedelta64[D]" ).astype(int)
 image_date["img_date_month"] = image_date["img_date"].dt.month
image_date["img_date_week"] = image_date["img_date"].dt.week
image_date["img_date_day"] = image_date["img_date"].dt.day
image_date["img_date_dayofweek"] = image_date["img_date"].dt.dayofweek
image_date["img_date_dayofyear"] = image_date["img_date"].dt.dayofyear
image_date["img_date_hour"] = image_date["img_date"].dt.hour
image_date["img_date_monthBeginMidEnd"] = image_date["img_date_day"].apply(lambda x: 1 if x<10 else 2 if x<20 else 3)
 train_df = pd.merge(train_df, image_date, on="listing_id", how="left")
 test_df = pd.merge(test_df, image_date, on="listing_id", how="left")
 features_to_use=["bathrooms", "bedrooms", "price_t","room_sum","latitude","longitude","num_photos","density","logprice","num_features","num_description_words","price_per_room","listing_id","img_date"]
print(train_df['price'].head())
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].str[:1]<categorify>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=321, num_rounds=818):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )<categorify>
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high')
 categorical = ["display_address", "manager_id", "building_id", "street_address"]
train["Cabin"].fillna(train.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test["Cabin"].fillna(test.groupby("Pclass")["Cabin"].transform("median"), inplace=True )<feature_engineering>
for f in categorical:
	if train_df[f].dtype=='object':
		
		lbl = preprocessing.LabelEncoder()
		lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
		train_df[f] = lbl.transform(list(train_df[f].values))
		test_df[f] = lbl.transform(list(test_df[f].values))
		features_to_use.append(f)
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1<categorify>
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
family_mapping = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].map(family_mapping )<drop_column>
def factorize(df1, df2, column):
ps = df1[column].append(df2[column])
 factors = ps.factorize() [0]
df1[column] = factors[:len(df1)]
df2[column] = factors[len(df1):]
return df1, df2
features_drop = ['Ticket', 'SibSp', 'Parch']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId'], axis=1 )<import_modules>
for col in('building_id', 'display_address', 'manager_id', 'street_address'):
train_df,test_df = factorize(train_df, test_df,col)
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import numpy as np<choose_model_class>
tfidf = CountVectorizer(stop_words='english', max_features=200)
 tfidf.fit(list(train_df['features'])+ list(test_df['features']))
tr_sparse = tfidf.transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
 print(te_sparse)
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )<compute_train_metric>
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
round(np.mean(score)*100, 2 )<compute_train_metric>
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape)
<compute_test_metric><EOS> 
preds, model = runXGB(train_X, train_y, test_X, num_rounds=818)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False)