Kaggle competition: New York City Taxi Fare Prediction
train["Embarked"].fillna("S", inplace = True)
 test["Embarked"].fillna("S", inplace = True )<categorify>
df = pd.read_csv('.. /input/train.csv',nrows =2000000)
 test_set = pd.read_csv(".. /input/test.csv" )
embark1 = pd.get_dummies(train['Embarked'])
 embark2 = pd.get_dummies(test['Embarked'])
 train.drop(['Embarked'],axis=1,inplace=True)
 test.drop(['Embarked'],axis=1,inplace=True)
 train = pd.concat([train,embark1],axis=1)
 test = pd.concat([test,embark2],axis=1 )<feature_engineering>
df = df.round(2)
 df = df.reindex(np.random.permutation(df.index))
def fam(x):
if(x['SibSp'] + x['Parch'])> 0:
return 1
else:
return 0
train['Family'] = train.apply(fam, axis = 1)
 test['Family'] = test.apply(fam, axis = 1 )<drop_column>
def distance(lat1, lon1, lat2, lon2):
distance = np.abs(lat2 - lat1)+ np.abs(lon2 - lon1)
 return distance
train = train.drop(['SibSp','Parch'],axis=1)
 test = test.drop(['SibSp','Parch'],axis=1 )<feature_engineering>
BB =(-75, -73, 40, 41.5)
 def select_within_boundingbox(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3])
 print('Old size: %d' % len(df))
df = df[select_within_boundingbox(df, BB)]
df = df[(df.passenger_count > 0)&(df.passenger_count <= 8)]
df = df[(df.fare_amount>0)&(df.fare_amount<=250)]
print('New size: %d' % len(df))
train["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in train['Cabin'] ])
 test["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in test['Cabin'] ] )<categorify>
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'])
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
dataset['year'] = dataset.pickup_datetime.dt.year
return dataset
df = add_datetime_info(df)
 test_set = add_datetime_info(test_set)
train["Cabin"] = train["Cabin"].map({"X":0, "A":1, "B" : 2 , "C":3, "D":4, "E":5, "F":6, "G":7,"T":0})
 train["Cabin"] = train["Cabin"].astype(int)
 test["Cabin"] = test["Cabin"].map({"X":0, "A":1, "B" : 2 , "C":3, "D":4, "E":5, "F":6, "G":7,"T":0})
 test["Cabin"] = test["Cabin"].astype(int )<feature_engineering>
train_title = [i.split(",")[1].split(".")[0].strip() for i in train["Name"]]
train["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title )<drop_column>
cor_frame = df.corr()
cor_frame.sort_values('fare_amount',axis=0 )
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )<categorify>
train = df.drop(['key','fare_amount','pickup_datetime'],axis=1)
 test = df['fare_amount']
X_train, X_test, y_train, y_test = train_test_split(train, test, test_size = 0.2 )
train["Title"] = train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train["Title"] = train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train["Title"] = train["Title"].astype(int)
 test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )<feature_engineering>
Ticket1 = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train["Ticket"] = Ticket1
Ticket2 = []
for j in list(test.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test["Ticket"] = Ticket2<concatenate>
dtrain = xgb.DMatrix(X_train, label=y_train)
 dtest = xgb.DMatrix(X_test)
 params = {'eval_metric': 'rmse',
'max_depth': 30,
'subsample': 0.8,
'eta': 0.1,
'gamma': 1,
'colsample_bytree': 1}
def xgb_evaluate(max_depth, gamma, colsample_bytree):
cv_result = xgb.cv(params, dtrain, num_boost_round=100, nfold=3)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]

 model2 = xgb.train(params, dtrain, num_boost_round=200)
np.union1d(train["Ticket"], test["Ticket"] )<categorify>
import pickle
train= pd.get_dummies(train, columns = ["Ticket"], prefix="T")
 test = pd.get_dummies(test, columns = ["Ticket"], prefix="T" )<drop_column>
predictions = model2.predict(dtest)
 predictions_train = model2.predict(dtrain )
train = train.drop(['T_SP','T_SOP','T_Fa','T_LINE','T_SWPP','T_SCOW','T_PPP','T_AS','T_CASOTON'],axis = 1)
 test = test.drop(['T_SCA3','T_STONOQ','T_AQ4','T_A','T_LP','T_AQ3'],axis = 1 )<drop_column>
rmse = math.sqrt(mean_squared_error(y_test,predictions))
rmse_train = math.sqrt(mean_squared_error(y_train,predictions_train))
print(rmse)
 print(rmse_train)
train.drop(['Survived'],axis=1,inplace=True )<count_missing_values>
print(train.isnull().sum())
 print("Number of columns are :",train.isnull().sum().count() )<count_missing_values>
test_set_features = test_set.drop(['key','pickup_datetime'],axis=1)
 test_set_key = test_set['key']
dtest_1 = xgb.DMatrix(test_set_features)
 y_pred_reg = model2.predict(dtest_1 )
<normalization><EOS> 
submission = pd.DataFrame(
 {'key': test_set_key, 'fare_amount': y_pred_reg},
columns = ['key', 'fare_amount'])
 submission = submission.round({'fare_amount':2})
 submission.to_csv('submission_combo.csv', index = False)
 print(submission.head())
 print("Submitted" )