Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
X_train = X_train.values.reshape(-1, 28, 28, 1)
 X_test = X_test.values.reshape(-1, 28, 28, 1)
 test = test.values.reshape(-1, 28, 28, 1 )<choose_model_class>
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json')
 train_df.head()
test_df["bathrooms"].loc[19671] = 1.5
test_df["bathrooms"].loc[22977] = 2.0
test_df["bathrooms"].loc[63719] = 2.0
train_df["price"] = train_df["price"].clip(upper=13000)
 test_df["price"] = test_df["price"].clip(upper=13000)
 ulimit = np.percentile(train_df.price.values, 99)
 train_df['price'].ix[train_df['price']>ulimit] = ulimit
ulimit = np.percentile(test_df.price.values, 99)
 test_df['price'].ix[test_df['price']>ulimit] = ulimit
train_df["logprice"] = np.log(train_df["price"])
 test_df["logprice"] = np.log(test_df["price"])
 train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["pos"] = train_df.longitude.round(3 ).astype(str)+ '_' + train_df.latitude.round(3 ).astype(str)
 test_df["pos"] = test_df.longitude.round(3 ).astype(str)+ '_' + test_df.latitude.round(3 ).astype(str)
 vals = train_df['pos'].value_counts()
dvals = vals.to_dict()
train_df["density"] = train_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
test_df["density"] = test_df['pos'].apply(lambda x: dvals.get(x, vals.min()))
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df["photo_per_room"]=train_df["num_photos"]/train_df["room_sum"]
test_df["photo_per_room"]=test_df["num_photos"]/test_df["room_sum"]
train_df["photo_per_price"]=train_df["num_photos"]/train_df["price"]
test_df["photo_per_price"]=test_df["num_photos"]/test_df["price"]
train_df['price_per_room'] = train_df['price']/train_df['room_sum']
test_df['price_per_room'] = test_df['price']/test_df['room_sum']
features_to_use=["bathrooms", "bedrooms", "price_t","room_sum","latitude","longitude","num_photos","density","logprice","num_features","num_description_words","price_per_room","listing_id","photo_per_room","photo_per_price"]
print(train_df['price'].head() )
model = Sequential()
model.add(Conv2D(64, kernel_size =(3, 3), activation = "relu", padding = "same", input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size =(3, 3), activation = "relu", padding = "same"))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128, kernel_size =(3, 3), activation = "relu", padding = "same"))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size =(3, 3), activation = "relu", padding = "same"))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(256, kernel_size =(3, 3), activation = "relu", padding = "same"))
model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size =(3, 3), activation = "relu", padding = "same"))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(512, activation = "relu"))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer = tf.keras.optimizers.RMSprop(lr = 0.0001, centered = True, momentum = 0),
loss = "categorical_crossentropy",
metrics = ["accuracy"])
 history = model.fit(X_train,
y_train,
validation_split = 0.2,
epochs = 80,
batch_size = 64,
shuffle = True,
verbose = 0)
 print("Test score :" + str(model.evaluate(X_test, y_test)))
 print("")
 print("Train score :" + str(model.evaluate(X_train, y_train)))
 print(model.summary() )<save_to_csv>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=321, num_rounds=730):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model
y_sub = pd.Series(np.argmax(model.predict(test), axis = 1), name = "Label")
 submission = pd.concat([pd.Series(range(1, 28001), name = "ImageId"), y_sub], axis = 1)
 submission.to_csv("submission.csv", index = False)
 submission.head()<set_options>
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]=[0,0,0]
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high')
 categorical = ["display_address", "manager_id", "building_id", "street_address"]
%matplotlib inline 
<define_variables>
for f in categorical:
	if train_df[f].dtype=='object':
		
		lbl = preprocessing.LabelEncoder()
		lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
		train_df[f] = lbl.transform(list(train_df[f].values))
		test_df[f] = lbl.transform(list(test_df[f].values))
		features_to_use.append(f )
img_rows, img_cols = 28, 28<load_from_csv>
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
df_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 df_test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 df_train.head()<split>
def factorize(df1, df2, column):
ps = df1[column].append(df2[column])
 factors = ps.factorize() [0]
df1[column] = factors[:len(df1)]
df2[column] = factors[len(df1):]
return df1, df2
y_train = df_train["label"]
X_train = df_train.drop(labels = ["label"],axis = 1)
 X_test = df_test
X_train /= 255
X_test /= 255
X_train = X_train.values.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.values.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=1)
 y_train = to_categorical(y_train, num_classes=10)
 y_val = to_categorical(y_val, num_classes=10 )<choose_model_class>
for col in('building_id', 'display_address', 'manager_id', 'street_address'):
train_df,test_df = factorize(train_df, test_df,col )
batch_size = 128
epochs = 200
lr = 1e-4
rho = 0.7
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.000001 )<choose_model_class>
tfidf = CountVectorizer(max_df=1, max_features=200)
 tfidf.fit(list(train_df['features'])+ list(test_df['features']))
tr_sparse = tfidf.transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
 print(te_sparse )
input_shape =(img_rows, img_cols, 1)
 num_classes = 10
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.compile(loss=categorical_crossentropy,
optimizer=RMSprop(learning_rate=lr, rho=rho),
metrics=['accuracy'] )<train_model>
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_val, y_val))
score = model.evaluate(X_val, y_val, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )<predict_on_test>
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )
<import_modules><EOS> 
preds, model = runXGB(train_X, train_y, test_X, num_rounds=730)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )