Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<feature_engineering>
warnings.filterwarnings("ignore" )
train_title = [i.split(",")[1].split(".")[0].strip() for i in train["Name"]]
train["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title )<drop_column>
original_train_data = pd.read_csv('.. /input/train.csv', nrows=6000000)
 train_data = original_train_data.sample(n=100000)
 train_data.info()
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )<categorify>
test_data = pd.read_csv('.. /input/test.csv')
 test_data.info()
train["Title"] = train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train["Title"] = train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train["Title"] = train["Title"].astype(int)
 test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )<feature_engineering>
train_data.isnull().sum()
Ticket1 = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train["Ticket"] = Ticket1
Ticket2 = []
for j in list(test.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test["Ticket"] = Ticket2<categorify>
train_data.dropna(axis=0,inplace=True)
train= pd.get_dummies(train, columns = ["Ticket"], prefix="T")
 test = pd.get_dummies(test, columns = ["Ticket"], prefix="T" )<drop_column>
train_data = train_data[train_data['fare_amount']>0]
train_data = train_data[(train_data['passenger_count']<=6)&(train_data['passenger_count']>0)]
train_data = train_data[(train_data['pickup_latitude']>-90)|(train_data['pickup_latitude']<=90)]
train_data = train_data[(train_data['dropoff_latitude']>-90)|(train_data['dropoff_latitude']<=90)]
train_data = train_data[(train_data['pickup_longitude']>=-180)|(train_data['pickup_longitude']<=180)]
train_data = train_data[(train_data['dropoff_longitude']>=-180)|(train_data['dropoff_longitude']<=180)]
train = train.drop(['T_SP','T_SOP','T_Fa','T_LINE','T_SWPP','T_SCOW','T_PPP','T_AS','T_CASOTON'],axis = 1)
 test = test.drop(['T_SCA3','T_STONOQ','T_AQ4','T_A','T_LP','T_AQ3'],axis = 1 )<drop_column>
def distance(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):

 R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def date_time_info(data):
data['pickup_datetime'] = pd.to_datetime(data['pickup_datetime'], format="%Y-%m-%d %H:%M:%S UTC")
 data['hour'] = data['pickup_datetime'].dt.hour
data['day'] = data['pickup_datetime'].dt.day
data['month'] = data['pickup_datetime'].dt.month
data['weekday'] = data['pickup_datetime'].dt.weekday
data['year'] = data['pickup_datetime'].dt.year
return data
train_data = date_time_info(train_data)
 train_data['distance'] = distance(train_data['pickup_latitude'],
train_data['pickup_longitude'],
train_data['dropoff_latitude'] ,
train_data['dropoff_longitude'])
 train_data.head()
train.drop(['Survived'],axis=1,inplace=True )<count_missing_values>
train_data.drop(['key', 'pickup_datetime'],axis =1, inplace = True)
 train_data.head()
print('Train:')
 print(train.isnull().sum())
 print('Test:')
 print(test.isnull().sum() )<normalization>
test_data = date_time_info(test_data)
 test_data['distance'] = distance(test_data['pickup_latitude'], test_data['pickup_longitude'],
test_data['dropoff_latitude'] , test_data['dropoff_longitude'])
 test_key = test_data['key']
x_pred = test_data.drop(columns=['key', 'pickup_datetime'] )
scaler = StandardScaler()
train2 = scaler.fit_transform(train)
 test2 = scaler.fit_transform(test )<create_dataframe>
y = train_data['fare_amount']
X = train_data.drop(['fare_amount'],axis=1)
 cv_split = KFold(n_splits=10,random_state=0)
KFold_Score = pd.DataFrame()
classifiers = ['Linear SVM', 'Radial SVM', 'LogisticRegression',
'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier', 'KNeighborsClassifier','GradientBoostingClassifier']
models = [svm.SVC(kernel='linear'),
svm.SVC(kernel='rbf'),
LogisticRegression(max_iter = 1000),
RandomForestClassifier(n_estimators=200, random_state=0),
AdaBoostClassifier(random_state = 0),
xgb.XGBClassifier(n_estimators=100),
KNeighborsClassifier() ,
GradientBoostingClassifier(random_state=0)
 ]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=0, shuffle=True)
 KFold_Score[classifiers[j]] =(cross_val_score(model, train2, np.ravel(pred), scoring = 'accuracy', cv=cv))
j = j+1<create_dataframe>
xgb = XGBRegressor(random_state=0)
 base_results = cross_validate(xgb, X,y, cv = cv_split)
 xgb.fit(X,y)
mean = pd.DataFrame(KFold_Score.mean() , index= classifiers)
 KFold_Score = pd.concat([KFold_Score,mean.T])
 KFold_Score.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
KFold_Score.T.sort_values(by=['Mean'], ascending = False )<feature_engineering>
params = {
'max_depth': [8],
'eta':[.03],
'subsample': [1],
'colsample_bytree': [0.8],
'objective':['reg:linear'],
'eval_metric':['rmse'],
'silent': [1]
}
submit_xgb = GridSearchCV(XGBRegressor() , param_grid=params,
scoring = 'neg_mean_squared_error',
cv = cv_split)
 submit_xgb.fit(X,y)
col_name1[0],col_name1[2] = col_name1[2],col_name1[0]
col_name2[0],col_name2[2] = col_name2[2],col_name2[0]<prepare_x_and_y>
prediction = submit_xgb.predict(x_pred )
train_new = train[col_name1]
test_new = test[col_name2]<drop_column>
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()