Kaggle competition: New York City Taxi Fare Prediction
class FillerTransformer() :

 def __init__(self, mean=True):

 self.mean = mean
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Embarked'] = mydf['Embarked'].fillna('C')
 mydf['Fare'].fillna(mydf['Fare'].median() , inplace = True)
 if self.mean:
mydf['Age'] = mydf.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.mean()))
else:
mydf['Age'] = mydf.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.median()))
return mydf<categorify>
train_df = pd.read_csv('.. /input/train.csv', nrows = 1_000_000)
 train_df.dtypes
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True))
])
 mytrain = preprocess.fit_transform(train)
 mytrain.Age.isnull().sum()<train_model>
print(train_df.isnull().sum() )
class ColumnsDelete() :

 def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
if self.columns:
mydf.drop(self.columns, axis=1, inplace=True)
 return mydf<categorify>
train_df = train_df.dropna(how = 'any', axis = 'rows' )
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket']))
])
 mytrain = preprocess.fit_transform(train)
 mytest = preprocess.fit_transform(test)
 mytrain.info() , mytest.info()<categorify>
def clean_df(df):
return df[(df.fare_amount > 0)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -70)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.dropoff_longitude > -80)&(df.dropoff_longitude < -70)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.passenger_count > 0)&(df.passenger_count < 10)]
train_df = clean_df(train_df)
 print(len(train_df))
class CategoryEncoder() :

 def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Sex'] = mydf['Sex'].map({'male': 1, 'female': 0})
 if self.columns:
mydf = pd.get_dummies(mydf, columns=self.columns, prefix=self.columns)
 return mydf<categorify>
def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):
R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'])
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
return dataset
train_df['distance'] = sphere_dist(train_df['pickup_latitude'], train_df['pickup_longitude'],
train_df['dropoff_latitude'] , train_df['dropoff_longitude'])
 train_df = add_datetime_info(train_df)
 train_df.head()
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket'])) , 
('CategoEncoder', CategoryEncoder(['Embarked', 'CabinDeck', 'Title'])) ,
])
 mytrain = preprocess.fit_transform(train)
 mytrain<train_model>
train_df.drop(columns=['key', 'pickup_datetime'], inplace=True)
 train_df.head()
class myZScaler(BaseEstimator, TransformerMixin):

 def __init__(self, columns = all, **init_params):
self.columns = columns
self.scaler = StandardScaler(**init_params)
 def fit(self, X, y=None):

 self.scaler.fit(X[self.columns], y)
 return self
def transform(self, X: pd):

 if self.columns is all:
self.columns = X.columns
init_col_order = X.columns
X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns, index = X.index)
 X_not_scaled = X.loc[:,~X.columns.isin(self.columns)]
return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]
myscaler = myZScaler(['Age', 'Fare'], copy=True)
 myfoo = myscaler.fit_transform(mytrain)
 myfoo.head()<categorify>
train_df['pickup_long_15'] = train_df['pickup_longitude']*np.cos(15* np.pi / 180)- train_df['pickup_latitude']*np.sin(15* np.pi/180)
 train_df['pickup_long_30'] = train_df['pickup_longitude']*np.cos(30* np.pi / 180)- train_df['pickup_latitude']*np.sin(30* np.pi/180)
 train_df['pickup_long_45'] = train_df['pickup_longitude']*np.cos(45* np.pi / 180)- train_df['pickup_latitude']*np.sin(45* np.pi/180)
 train_df['pickup_long_60'] = train_df['pickup_longitude']*np.cos(60* np.pi / 180)- train_df['pickup_latitude']*np.sin(60* np.pi/180)
 train_df['pickup_long_75'] = train_df['pickup_longitude']*np.cos(75* np.pi / 180)- train_df['pickup_latitude']*np.sin(75* np.pi/180)
 train_df['pickup_lat_15'] = train_df['pickup_longitude']*np.sin(15* np.pi / 180)+ train_df['pickup_latitude']*np.cos(15* np.pi/180)
 train_df['pickup_lat_30'] = train_df['pickup_longitude']*np.sin(30* np.pi / 180)+ train_df['pickup_latitude']*np.cos(30* np.pi/180)
 train_df['pickup_lat_45'] = train_df['pickup_longitude']*np.sin(45* np.pi / 180)+ train_df['pickup_latitude']*np.cos(45* np.pi/180)
 train_df['pickup_lat_60'] = train_df['pickup_longitude']*np.sin(60* np.pi / 180)+ train_df['pickup_latitude']*np.cos(60* np.pi/180)
 train_df['pickup_lat_75'] = train_df['pickup_longitude']*np.sin(75* np.pi / 180)+ train_df['pickup_latitude']*np.cos(75* np.pi/180)
 train_df['dropoff_long_15'] = train_df['dropoff_longitude']*np.cos(15* np.pi / 180)- train_df['dropoff_latitude']*np.sin(15* np.pi/180)
 train_df['dropoff_long_30'] = train_df['dropoff_longitude']*np.cos(30* np.pi / 180)- train_df['dropoff_latitude']*np.sin(30* np.pi/180)
 train_df['dropoff_long_45'] = train_df['dropoff_longitude']*np.cos(45* np.pi / 180)- train_df['dropoff_latitude']*np.sin(45* np.pi/180)
 train_df['dropoff_long_60'] = train_df['dropoff_longitude']*np.cos(60* np.pi / 180)- train_df['dropoff_latitude']*np.sin(60* np.pi/180)
 train_df['dropoff_long_75'] = train_df['dropoff_longitude']*np.cos(75* np.pi / 180)- train_df['dropoff_latitude']*np.sin(75* np.pi/180)
 train_df['dropoff_lat_15'] = train_df['dropoff_longitude']*np.sin(15* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(15* np.pi/180)
 train_df['dropoff_lat_30'] = train_df['dropoff_longitude']*np.sin(30* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(30* np.pi/180)
 train_df['dropoff_lat_45'] = train_df['dropoff_longitude']*np.sin(45* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(45* np.pi/180)
 train_df['dropoff_lat_60'] = train_df['dropoff_longitude']*np.sin(60* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(60* np.pi/180)
 train_df['dropoff_lat_75'] = train_df['dropoff_longitude']*np.sin(75* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(75* np.pi/180 )
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=False)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket'])) , 
('CategoEncoder', CategoryEncoder(['Embarked', 'CabinDeck', 'Title'])) , 
('z-score' , myZScaler(['Age', 'Fare', 'FamilySize', 'Pclass'], copy=True))
])
 train.loc[797,'Name'] = "Leader, Ms.Alice(Farnham)"
mytrain = preprocess.fit_transform(train )<categorify>
y = train_df['fare_amount']
train = train_df.drop(columns=['fare_amount'])
 x_train,x_test,y_train,y_test = train_test_split(train,y,random_state=0,test_size=0.2 )
mytrain = preprocess.fit_transform(train)
 mytest = preprocess.fit_transform(test )<prepare_x_and_y>
def XGBmodel(x_train,x_test,y_train,y_test):
matrix_train = xgb.DMatrix(x_train,label=y_train)
 matrix_test = xgb.DMatrix(x_test,label=y_test)
 model=xgb.train(params={'objective':'reg:linear','eval_metric':'rmse'},
dtrain=matrix_train,num_boost_round=100,
early_stopping_rounds=100,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(x_train,x_test,y_train,y_test )
target = mytrain['Survived']
mytrain.drop(['Survived'], axis=1, inplace=True )<train_model>
test_df = pd.read_csv('.. /input/test.csv')
 test_df['distance'] = sphere_dist(test_df['pickup_latitude'], test_df['pickup_longitude'],
test_df['dropoff_latitude'] , test_df['dropoff_longitude'])
 test_df = add_datetime_info(test_df)
 test_df['pickup_long_15'] = test_df['pickup_longitude']*np.cos(15* np.pi / 180)- test_df['pickup_latitude']*np.sin(15* np.pi/180)
 test_df['pickup_long_30'] = test_df['pickup_longitude']*np.cos(30* np.pi / 180)- test_df['pickup_latitude']*np.sin(30* np.pi/180)
 test_df['pickup_long_45'] = test_df['pickup_longitude']*np.cos(45* np.pi / 180)- test_df['pickup_latitude']*np.sin(45* np.pi/180)
 test_df['pickup_long_60'] = test_df['pickup_longitude']*np.cos(60* np.pi / 180)- test_df['pickup_latitude']*np.sin(60* np.pi/180)
 test_df['pickup_long_75'] = test_df['pickup_longitude']*np.cos(75* np.pi / 180)- test_df['pickup_latitude']*np.sin(75* np.pi/180)
 test_df['pickup_lat_15'] = test_df['pickup_longitude']*np.sin(15* np.pi / 180)+ test_df['pickup_latitude']*np.cos(15* np.pi/180)
 test_df['pickup_lat_30'] = test_df['pickup_longitude']*np.sin(30* np.pi / 180)+ test_df['pickup_latitude']*np.cos(30* np.pi/180)
 test_df['pickup_lat_45'] = test_df['pickup_longitude']*np.sin(45* np.pi / 180)+ test_df['pickup_latitude']*np.cos(45* np.pi/180)
 test_df['pickup_lat_60'] = test_df['pickup_longitude']*np.sin(60* np.pi / 180)+ test_df['pickup_latitude']*np.cos(60* np.pi/180)
 test_df['pickup_lat_75'] = test_df['pickup_longitude']*np.sin(75* np.pi / 180)+ test_df['pickup_latitude']*np.cos(75* np.pi/180)
 test_df['dropoff_long_15'] = test_df['dropoff_longitude']*np.cos(15* np.pi / 180)- test_df['dropoff_latitude']*np.sin(15* np.pi/180)
 test_df['dropoff_long_30'] = test_df['dropoff_longitude']*np.cos(30* np.pi / 180)- test_df['dropoff_latitude']*np.sin(30* np.pi/180)
 test_df['dropoff_long_45'] = test_df['dropoff_longitude']*np.cos(45* np.pi / 180)- test_df['dropoff_latitude']*np.sin(45* np.pi/180)
 test_df['dropoff_long_60'] = test_df['dropoff_longitude']*np.cos(60* np.pi / 180)- test_df['dropoff_latitude']*np.sin(60* np.pi/180)
 test_df['dropoff_long_75'] = test_df['dropoff_longitude']*np.cos(75* np.pi / 180)- test_df['dropoff_latitude']*np.sin(75* np.pi/180)
 test_df['dropoff_lat_15'] = test_df['dropoff_longitude']*np.sin(15* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(15* np.pi/180)
 test_df['dropoff_lat_30'] = test_df['dropoff_longitude']*np.sin(30* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(30* np.pi/180)
 test_df['dropoff_lat_45'] = test_df['dropoff_longitude']*np.sin(45* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(45* np.pi/180)
 test_df['dropoff_lat_60'] = test_df['dropoff_longitude']*np.sin(60* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(60* np.pi/180)
 test_df['dropoff_lat_75'] = test_df['dropoff_longitude']*np.sin(75* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(75* np.pi/180)
 test_key = test_df['key']
x_pred = test_df.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit = model.best_ntree_limit )
<compute_test_metric><EOS> 
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission