Kaggle competition: New York City Taxi Fare Prediction
gbm.fit(X_train_df, y_train_df )<predict_on_test>
nyc_weather = pd.read_csv('.. /input/nyc-weather/nyc_weather.csv')
 weather_cols = ['DATE','AWND','PRCP','SNOW','TMAX','TMIN']
nyc_weather = nyc_weather[weather_cols].copy()
nyc_weather['DATE'] = pd.to_datetime(nyc_weather['DATE'], utc=True, format='%m/%d/%Y')
 nyc_weather.head()
X_test = test_df
y_test_gbm = gbm.predict(X_test )<save_to_csv>
holidays = pd.read_csv('.. /input/us-bank-holidays-20092018/US Bank Holidays 2012-2018.csv')
 holidays['Date'] = pd.to_datetime(holidays['Date'], utc=True, format='%m/%d/%y')
 holidays.head(12 )
lasso_submission = pd.DataFrame({'id': test_df['id'], 'target': y_test_lasso})
 lasso_submission.to_csv('lasso_submission.csv', index=False )<save_to_csv>
%%time
n = sum(1 for line in open('.. /input/new-york-city-taxi-fare-prediction/train.csv')) - 1
s = 10000000
skip = sorted(random.sample(range(1,n+1),n-s))
train_full = pd.read_csv('.. /input/new-york-city-taxi-fare-prediction/train.csv', skiprows=skip)
 test = pd.read_csv('.. /input/new-york-city-taxi-fare-prediction/test.csv')
 test_id = test.key.values
ridge_submission = pd.DataFrame({'id': test_df['id'], 'target': y_test_ridge})
 ridge_submission.to_csv('ridge_submission.csv', index=False )<save_to_csv>
train_one = train_full.iloc[:2500000,:]
train_two = train_full.iloc[2500000:5000000,:]
train_three = train_full.iloc[5000000:7500000,:]
train_four = train_full.iloc[7500000:,:]
train_one.head()
lgbm_submission = pd.DataFrame({'id': test_df['id'], 'target': y_test_lgbm})
 lgbm_submission.to_csv('lgbm_submission.csv', index=False )<save_to_csv>
%%time
train_sample = train_one.dropna()
train_sample['pickup_datetime'] = train_sample.pickup_datetime.str.slice(0, 16)
 test['pickup_datetime'] = test.pickup_datetime.str.slice(0, 16)
 train_sample['pickup_datetime'] = pd.to_datetime(train_sample.pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime, utc=True,format='%Y-%m-%d %H:%M')
 train_sample.drop(labels='key', axis=1, inplace=True)
 test.drop(labels='key', axis=1, inplace=True)
 train_sample.loc[:,'passenger_count'] = train_sample.passenger_count.astype(dtype = 'uint8')
 train_sample['pickup_longitude'] = train_sample.pickup_longitude.astype(dtype = 'float32')
 train_sample['pickup_latitude'] = train_sample.pickup_latitude.astype(dtype = 'float32')
 train_sample['dropoff_longitude'] = train_sample.dropoff_longitude.astype(dtype = 'float32')
 train_sample['dropoff_latitude'] = train_sample.dropoff_latitude.astype(dtype = 'float32')
 train_sample['fare_amount'] = train_sample.fare_amount.astype(dtype = 'float32')
 test['pickup_longitude'] = test.pickup_longitude.astype(dtype = 'float32')
 test['pickup_latitude'] = test.pickup_latitude.astype(dtype = 'float32')
 test['dropoff_longitude'] = test.dropoff_longitude.astype(dtype = 'float32')
 test['dropoff_latitude'] = test.dropoff_latitude.astype(dtype = 'float32')
 train_sample = train_sample.loc[train_sample.pickup_longitude.between(test.pickup_longitude.min() , test.pickup_longitude.max())]
train_sample = train_sample.loc[train_sample.pickup_latitude.between(test.pickup_latitude.min() , test.pickup_latitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_longitude.between(test.dropoff_longitude.min() , test.dropoff_longitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_latitude.between(test.dropoff_latitude.min() , test.dropoff_latitude.max())]
train_sample['hour'] = train_sample['pickup_datetime'].apply(lambda time: time.hour)
 train_sample['month'] = train_sample['pickup_datetime'].apply(lambda time: time.month)
 train_sample['day_of_week'] = train_sample['pickup_datetime'].apply(lambda time: time.dayofweek)
 train_sample['year'] = train_sample['pickup_datetime'].apply(lambda t: t.year)
 test['hour'] = test['pickup_datetime'].apply(lambda time: time.hour)
 test['month'] = test['pickup_datetime'].apply(lambda time: time.month)
 test['day_of_week'] = test['pickup_datetime'].apply(lambda time: time.dayofweek)
 test['year'] = test['pickup_datetime'].apply(lambda t: t.year)
 train_sample['hour'] = train_sample.hour.astype(dtype = 'uint8')
 train_sample['month'] = train_sample.month.astype(dtype = 'uint8')
 train_sample['day_of_week'] = train_sample.day_of_week.astype(dtype = 'uint8')
 train_sample['year'] = train_sample.year.astype(dtype = 'uint16')
 test['hour'] = test.hour.astype(dtype = 'uint8')
 test['month'] = test.month.astype(dtype = 'uint8')
 test['day_of_week'] = test.day_of_week.astype(dtype = 'uint8')
 test['year'] = test.year.astype(dtype = 'uint16')
 train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(nyc_weather, how = 'left', left_on ='pickup_day', right_on = 'DATE')
 train_sample.drop(columns = ['pickup_day','DATE'], axis = 0, inplace = True)
 test['pickup_day'] = test.pickup_datetime.dt.floor('d')
 test = test.merge(nyc_weather, how = 'left', left_on ='pickup_day', right_on = 'DATE')
 test.drop(columns = ['pickup_day','DATE'], axis = 0, inplace = True)
 train_sample['AWND'] = train_sample.AWND.astype(dtype = 'float16')
 train_sample['PRCP'] = train_sample.PRCP.astype(dtype = 'float16')
 train_sample['SNOW'] = train_sample.day_of_week.astype(dtype = 'float16')
 train_sample['TMAX'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['TMIN'] = train_sample.TMAX.astype(dtype = 'float16')
 test['AWND'] = test.AWND.astype(dtype = 'float16')
 test['PRCP'] = test.PRCP.astype(dtype = 'float16')
 test['SNOW'] = test.day_of_week.astype(dtype = 'float16')
 test['TMAX'] = test.TMAX.astype(dtype = 'float16')
 test['TMIN'] = test.TMAX.astype(dtype = 'float16')
 train_sample['hot_day'] = np.where(train_sample.TMAX >= 30,1,0)
 train_sample['cold_day'] = np.where(train_sample.TMIN <= 0,1,0)
 test['hot_day'] = np.where(test.TMAX >= 30,1,0)
 test['cold_day'] = np.where(test.TMIN <= 0,1,0)
 train_sample['hot_day'] = train_sample.hot_day.astype(dtype = 'uint8')
 train_sample['cold_day'] = train_sample.cold_day.astype(dtype = 'uint8')
 test['hot_day'] = test.hot_day.astype(dtype = 'uint8')
 test['cold_day'] = test.cold_day.astype(dtype = 'uint8')
 train_sample['rainy_day'] = np.where(train_sample.PRCP >= 0,1,0)
 train_sample['snowy_day'] = np.where(train_sample.SNOW <= 0,1,0)
 test['rainy_day'] = np.where(test.PRCP >= 0,1,0)
 test['snowy_day'] = np.where(test.SNOW <= 0,1,0)
 train_sample['rainy_day'] = train_sample.rainy_day.astype(dtype = 'uint8')
 train_sample['snowy_day'] = train_sample.snowy_day.astype(dtype = 'uint8')
 test['rainy_day'] = test.rainy_day.astype(dtype = 'uint8')
 test['snowy_day'] = test.snowy_day.astype(dtype = 'uint8')
 train_sample['windy_day'] = np.where(train_sample.AWND >= 0,1,0)
 test['windy_day'] = np.where(test.AWND >= 0,1,0)
 train_sample['windy_day'] = train_sample.windy_day.astype(dtype = 'uint8')
 test['windy_day'] = test.windy_day.astype(dtype = 'uint8')
 def degree_to_radion(degree):
return degree*(np.pi/180)
 def calculate_distance(pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude):
from_lat = degree_to_radion(pickup_latitude)
 from_long = degree_to_radion(pickup_longitude)
 to_lat = degree_to_radion(dropoff_latitude)
 to_long = degree_to_radion(dropoff_longitude)
 radius = 6371.01
lat_diff = to_lat - from_lat
long_diff = to_long - from_long
a = np.sin(lat_diff / 2)**2 + np.cos(degree_to_radion(from_lat)) * np.cos(degree_to_radion(to_lat)) * np.sin(long_diff / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
return radius * c
train_sample['distance'] = calculate_distance(train_sample.pickup_latitude, train_sample.pickup_longitude, train_sample.dropoff_latitude, train_sample.dropoff_longitude)
 test['distance'] = calculate_distance(test.pickup_latitude, test.pickup_longitude, test.dropoff_latitude, test.dropoff_longitude)
 train_sample['distance'] = train_sample.distance.astype(dtype = 'float32')
 test['distance'] = test.distance.astype(dtype = 'float32')
 train_sample['day_hour'] = train_sample.day_of_week.astype(str)+ "_" + train_sample.hour.astype(str)
 train_sample['day_hour'] = train_sample['day_hour'].astype('category')
 test['day_hour'] = test.day_of_week.astype(str)+ test.hour.astype(str)
 test['day_hour'] = test['day_hour'].astype('category')
 train_sample = train_sample[train_sample.fare_amount > 0]
train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(holidays, left_on = 'pickup_day', right_on = 'Date', how = 'left')
 train_sample['Holiday'] =train_sample.Holiday.fillna('None')
 le = LabelEncoder()
train_sample['holiday'] = le.fit_transform(train_sample.Holiday.values)
 train_sample.drop(['Holiday','Date','pickup_day'], axis = 1, inplace = True)
 test['pickup_day'] = test.pickup_datetime.dt.floor('d')
 test = test.merge(holidays, left_on = 'pickup_day', right_on = 'Date', how = 'left')
 test['Holiday'] =test.Holiday.fillna('None')
 test['holiday'] = le.fit_transform(test.Holiday.values)
 test.drop(['Holiday','Date','pickup_day'], axis = 1, inplace = True)
 train_sample['holiday'] = train_sample.holiday.astype(dtype = 'uint8')
 test['holiday'] = test.holiday.astype(dtype = 'uint8')
 train_one = train_sample.copy()
del train_sample
train_one.head()
ada_submission = pd.DataFrame({'id': test_df['id'], 'target': y_test_ada})
 ada_submission.to_csv('ada_submission.csv', index=False )<save_to_csv>
%%time
train_sample = train_two.dropna()
train_sample['pickup_datetime'] = train_sample.pickup_datetime.str.slice(0, 16)
 train_sample['pickup_datetime'] = pd.to_datetime(train_sample.pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 train_sample.drop(labels='key', axis=1, inplace=True)
 train_sample.loc[:,'passenger_count'] = train_sample.passenger_count.astype(dtype = 'uint8')
 train_sample['pickup_longitude'] = train_sample.pickup_longitude.astype(dtype = 'float32')
 train_sample['pickup_latitude'] = train_sample.pickup_latitude.astype(dtype = 'float32')
 train_sample['dropoff_longitude'] = train_sample.dropoff_longitude.astype(dtype = 'float32')
 train_sample['dropoff_latitude'] = train_sample.dropoff_latitude.astype(dtype = 'float32')
 train_sample['fare_amount'] = train_sample.fare_amount.astype(dtype = 'float32')
 train_sample = train_sample.loc[train_sample.pickup_longitude.between(test.pickup_longitude.min() , test.pickup_longitude.max())]
train_sample = train_sample.loc[train_sample.pickup_latitude.between(test.pickup_latitude.min() , test.pickup_latitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_longitude.between(test.dropoff_longitude.min() , test.dropoff_longitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_latitude.between(test.dropoff_latitude.min() , test.dropoff_latitude.max())]
train_sample['hour'] = train_sample['pickup_datetime'].apply(lambda time: time.hour)
 train_sample['month'] = train_sample['pickup_datetime'].apply(lambda time: time.month)
 train_sample['day_of_week'] = train_sample['pickup_datetime'].apply(lambda time: time.dayofweek)
 train_sample['year'] = train_sample['pickup_datetime'].apply(lambda t: t.year)
 train_sample['hour'] = train_sample.hour.astype(dtype = 'uint8')
 train_sample['month'] = train_sample.month.astype(dtype = 'uint8')
 train_sample['day_of_week'] = train_sample.day_of_week.astype(dtype = 'uint8')
 train_sample['year'] = train_sample.year.astype(dtype = 'uint16')
 train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(nyc_weather, how = 'left', left_on ='pickup_day', right_on = 'DATE')
 train_sample.drop(columns = ['pickup_day','DATE'], axis = 0, inplace = True)
 train_sample['AWND'] = train_sample.AWND.astype(dtype = 'float16')
 train_sample['PRCP'] = train_sample.PRCP.astype(dtype = 'float16')
 train_sample['SNOW'] = train_sample.day_of_week.astype(dtype = 'float16')
 train_sample['TMAX'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['TMIN'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['hot_day'] = np.where(train_sample.TMAX >= 30,1,0)
 train_sample['cold_day'] = np.where(train_sample.TMIN <= 0,1,0)
 train_sample['hot_day'] = train_sample.hot_day.astype(dtype = 'uint8')
 train_sample['cold_day'] = train_sample.cold_day.astype(dtype = 'uint8')
 train_sample['rainy_day'] = np.where(train_sample.PRCP >= 0,1,0)
 train_sample['snowy_day'] = np.where(train_sample.SNOW <= 0,1,0)
 train_sample['rainy_day'] = train_sample.rainy_day.astype(dtype = 'uint8')
 train_sample['snowy_day'] = train_sample.snowy_day.astype(dtype = 'uint8')
 train_sample['windy_day'] = np.where(train_sample.AWND >= 0,1,0)
 train_sample['windy_day'] = train_sample.windy_day.astype(dtype = 'uint8')
 train_sample['distance'] = calculate_distance(train_sample.pickup_latitude, train_sample.pickup_longitude, train_sample.dropoff_latitude, train_sample.dropoff_longitude)
 train_sample['distance'] = train_sample.distance.astype(dtype = 'float32')
 train_sample['day_hour'] = train_sample.day_of_week.astype(str)+ "_" + train_sample.hour.astype(str)
 train_sample['day_hour'] = train_sample['day_hour'].astype('category')
 train_sample = train_sample[train_sample.fare_amount > 0]
train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(holidays, left_on = 'pickup_day', right_on = 'Date', how = 'left')
 train_sample['Holiday'] =train_sample.Holiday.fillna('None')
 le = LabelEncoder()
train_sample['holiday'] = le.fit_transform(train_sample.Holiday.values)
 train_sample.drop(['Holiday','Date','pickup_day'], axis = 1, inplace = True)
 train_sample['holiday'] = train_sample.holiday.astype(dtype = 'uint8')
 train_two = train_sample.copy()
del train_sample
train_two.head()
gbm_submission = pd.DataFrame({'id': test_df['id'], 'target': y_test_gbm})
 gbm_submission.to_csv('gbm_submission.csv', index=False )<import_modules>
%%time
train_sample = train_three.dropna()
train_sample['pickup_datetime'] = train_sample.pickup_datetime.str.slice(0, 16)
 train_sample['pickup_datetime'] = pd.to_datetime(train_sample.pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 train_sample.drop(labels='key', axis=1, inplace=True)
 train_sample.loc[:,'passenger_count'] = train_sample.passenger_count.astype(dtype = 'uint8')
 train_sample['pickup_longitude'] = train_sample.pickup_longitude.astype(dtype = 'float32')
 train_sample['pickup_latitude'] = train_sample.pickup_latitude.astype(dtype = 'float32')
 train_sample['dropoff_longitude'] = train_sample.dropoff_longitude.astype(dtype = 'float32')
 train_sample['dropoff_latitude'] = train_sample.dropoff_latitude.astype(dtype = 'float32')
 train_sample['fare_amount'] = train_sample.fare_amount.astype(dtype = 'float32')
 train_sample = train_sample.loc[train_sample.pickup_longitude.between(test.pickup_longitude.min() , test.pickup_longitude.max())]
train_sample = train_sample.loc[train_sample.pickup_latitude.between(test.pickup_latitude.min() , test.pickup_latitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_longitude.between(test.dropoff_longitude.min() , test.dropoff_longitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_latitude.between(test.dropoff_latitude.min() , test.dropoff_latitude.max())]
train_sample['hour'] = train_sample['pickup_datetime'].apply(lambda time: time.hour)
 train_sample['month'] = train_sample['pickup_datetime'].apply(lambda time: time.month)
 train_sample['day_of_week'] = train_sample['pickup_datetime'].apply(lambda time: time.dayofweek)
 train_sample['year'] = train_sample['pickup_datetime'].apply(lambda t: t.year)
 train_sample['hour'] = train_sample.hour.astype(dtype = 'uint8')
 train_sample['month'] = train_sample.month.astype(dtype = 'uint8')
 train_sample['day_of_week'] = train_sample.day_of_week.astype(dtype = 'uint8')
 train_sample['year'] = train_sample.year.astype(dtype = 'uint16')
 train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(nyc_weather, how = 'left', left_on ='pickup_day', right_on = 'DATE')
 train_sample.drop(columns = ['pickup_day','DATE'], axis = 0, inplace = True)
 train_sample['AWND'] = train_sample.AWND.astype(dtype = 'float16')
 train_sample['PRCP'] = train_sample.PRCP.astype(dtype = 'float16')
 train_sample['SNOW'] = train_sample.day_of_week.astype(dtype = 'float16')
 train_sample['TMAX'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['TMIN'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['hot_day'] = np.where(train_sample.TMAX >= 30,1,0)
 train_sample['cold_day'] = np.where(train_sample.TMIN <= 0,1,0)
 train_sample['hot_day'] = train_sample.hot_day.astype(dtype = 'uint8')
 train_sample['cold_day'] = train_sample.cold_day.astype(dtype = 'uint8')
 train_sample['rainy_day'] = np.where(train_sample.PRCP >= 0,1,0)
 train_sample['snowy_day'] = np.where(train_sample.SNOW <= 0,1,0)
 train_sample['rainy_day'] = train_sample.rainy_day.astype(dtype = 'uint8')
 train_sample['snowy_day'] = train_sample.snowy_day.astype(dtype = 'uint8')
 train_sample['windy_day'] = np.where(train_sample.AWND >= 0,1,0)
 train_sample['windy_day'] = train_sample.windy_day.astype(dtype = 'uint8')
 train_sample['distance'] = calculate_distance(train_sample.pickup_latitude, train_sample.pickup_longitude, train_sample.dropoff_latitude, train_sample.dropoff_longitude)
 train_sample['distance'] = train_sample.distance.astype(dtype = 'float32')
 train_sample['day_hour'] = train_sample.day_of_week.astype(str)+ "_" + train_sample.hour.astype(str)
 train_sample['day_hour'] = train_sample['day_hour'].astype('category')
 train_sample = train_sample[train_sample.fare_amount > 0]
train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(holidays, left_on = 'pickup_day', right_on = 'Date', how = 'left')
 train_sample['Holiday'] =train_sample.Holiday.fillna('None')
 le = LabelEncoder()
train_sample['holiday'] = le.fit_transform(train_sample.Holiday.values)
 train_sample.drop(['Holiday','Date','pickup_day'], axis = 1, inplace = True)
 train_sample['holiday'] = train_sample.holiday.astype(dtype = 'uint8')
 train_three = train_sample.copy()
del train_sample
train_three.head()
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from tqdm import tqdm
import pandas as pd
import xgboost as xgb<load_from_csv>
%%time
train_sample = train_four.dropna()
train_sample['pickup_datetime'] = train_sample.pickup_datetime.str.slice(0, 16)
 train_sample['pickup_datetime'] = pd.to_datetime(train_sample.pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 train_sample.drop(labels='key', axis=1, inplace=True)
 train_sample.loc[:,'passenger_count'] = train_sample.passenger_count.astype(dtype = 'uint8')
 train_sample['pickup_longitude'] = train_sample.pickup_longitude.astype(dtype = 'float32')
 train_sample['pickup_latitude'] = train_sample.pickup_latitude.astype(dtype = 'float32')
 train_sample['dropoff_longitude'] = train_sample.dropoff_longitude.astype(dtype = 'float32')
 train_sample['dropoff_latitude'] = train_sample.dropoff_latitude.astype(dtype = 'float32')
 train_sample['fare_amount'] = train_sample.fare_amount.astype(dtype = 'float32')
 train_sample = train_sample.loc[train_sample.pickup_longitude.between(test.pickup_longitude.min() , test.pickup_longitude.max())]
train_sample = train_sample.loc[train_sample.pickup_latitude.between(test.pickup_latitude.min() , test.pickup_latitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_longitude.between(test.dropoff_longitude.min() , test.dropoff_longitude.max())]
train_sample = train_sample.loc[train_sample.dropoff_latitude.between(test.dropoff_latitude.min() , test.dropoff_latitude.max())]
train_sample['hour'] = train_sample['pickup_datetime'].apply(lambda time: time.hour)
 train_sample['month'] = train_sample['pickup_datetime'].apply(lambda time: time.month)
 train_sample['day_of_week'] = train_sample['pickup_datetime'].apply(lambda time: time.dayofweek)
 train_sample['year'] = train_sample['pickup_datetime'].apply(lambda t: t.year)
 train_sample['hour'] = train_sample.hour.astype(dtype = 'uint8')
 train_sample['month'] = train_sample.month.astype(dtype = 'uint8')
 train_sample['day_of_week'] = train_sample.day_of_week.astype(dtype = 'uint8')
 train_sample['year'] = train_sample.year.astype(dtype = 'uint16')
 train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(nyc_weather, how = 'left', left_on ='pickup_day', right_on = 'DATE')
 train_sample.drop(columns = ['pickup_day','DATE'], axis = 0, inplace = True)
 train_sample['AWND'] = train_sample.AWND.astype(dtype = 'float16')
 train_sample['PRCP'] = train_sample.PRCP.astype(dtype = 'float16')
 train_sample['SNOW'] = train_sample.day_of_week.astype(dtype = 'float16')
 train_sample['TMAX'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['TMIN'] = train_sample.TMAX.astype(dtype = 'float16')
 train_sample['hot_day'] = np.where(train_sample.TMAX >= 30,1,0)
 train_sample['cold_day'] = np.where(train_sample.TMIN <= 0,1,0)
 train_sample['hot_day'] = train_sample.hot_day.astype(dtype = 'uint8')
 train_sample['cold_day'] = train_sample.cold_day.astype(dtype = 'uint8')
 train_sample['rainy_day'] = np.where(train_sample.PRCP >= 0,1,0)
 train_sample['snowy_day'] = np.where(train_sample.SNOW <= 0,1,0)
 train_sample['rainy_day'] = train_sample.rainy_day.astype(dtype = 'uint8')
 train_sample['snowy_day'] = train_sample.snowy_day.astype(dtype = 'uint8')
 train_sample['windy_day'] = np.where(train_sample.AWND >= 0,1,0)
 train_sample['windy_day'] = train_sample.windy_day.astype(dtype = 'uint8')
 train_sample['distance'] = calculate_distance(train_sample.pickup_latitude, train_sample.pickup_longitude, train_sample.dropoff_latitude, train_sample.dropoff_longitude)
 train_sample['distance'] = train_sample.distance.astype(dtype = 'float32')
 train_sample['day_hour'] = train_sample.day_of_week.astype(str)+ "_" + train_sample.hour.astype(str)
 train_sample['day_hour'] = train_sample['day_hour'].astype('category')
 train_sample = train_sample[train_sample.fare_amount > 0]
train_sample['pickup_day'] = train_sample.pickup_datetime.dt.floor('d')
 train_sample = train_sample.merge(holidays, left_on = 'pickup_day', right_on = 'Date', how = 'left')
 train_sample['Holiday'] =train_sample.Holiday.fillna('None')
 le = LabelEncoder()
train_sample['holiday'] = le.fit_transform(train_sample.Holiday.values)
 train_sample.drop(['Holiday','Date','pickup_day'], axis = 1, inplace = True)
 train_sample['holiday'] = train_sample.holiday.astype(dtype = 'uint8')
 train_four = train_sample.copy()
del train_sample
train_four.head()
train_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 train_data.head()<prepare_x_and_y>
train_full_pre = pd.concat([train_one, train_two, train_three,train_four] )
features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
X_train = train_data[features]
y_train = train_data["target"]
final_X_test = test_data[features]<split>
del train_full
X_train,X_test,y_train,y_test = train_test_split(X_train, y_train, test_size=0.1 )<init_hyperparams>
train_sample = train_full_pre.copy()
full_pickups = pd.concat([train_sample[['pickup_longitude','pickup_latitude']],test[['pickup_longitude','pickup_latitude']]], axis = 0)
 full_pickups.columns = ['x','y']
full_dropoffs = pd.concat([train_sample[['dropoff_longitude','dropoff_latitude']],test[['dropoff_longitude','dropoff_latitude']]], axis = 0)
 full_dropoffs.columns = ['x','y']
full_locs = pd.concat([full_pickups,full_dropoffs], axis = 0)
 full_locs = full_locs.groupby(['x','y'] ).count().reset_index()
del full_pickups, full_dropoffs, train_sample, train_full_pre
full_locs.info()
Best_trial = {'lambda': 0.0030282073258141168, 'alpha': 0.01563845128469084, 'colsample_bytree': 0.5,
'subsample': 0.7,'n_estimators': 4000, 'learning_rate': 0.01,'max_depth': 15,
'random_state': 2020, 'min_child_weight': 257,'tree_method':'gpu_hist'
,'predictor': 'gpu_predictor'}<train_model>
%%time
X_df = full_locs.copy()
X_kmeans = full_locs.values
del full_locs
num_clusters = 200
with open('.. /input/taxi-weather-holidays-kmeans-neighborhoods/kmeans_200_round4.pkl', 'rb')as fid:
kmeans = pickle.load(fid)
 z = kmeans.predict(X_kmeans)
 centers = kmeans.cluster_centers_
x_centers = [pair[0] for pair in centers]
y_centers = [pair[1] for pair in centers]
z_centers = np.arange(num_clusters)
 plt.subplot(1,2,1)
 plt.scatter(X_df['x'], X_df['y'], c=z)
 plt.gray()
plt.xlabel('Pickup/Dropoff Longitude')
 plt.ylabel('Pickup/Dropoff Latitude')
 plt.title('Clusters of NYC locations')
 plt.subplot(1,2,2)
 plt.scatter(x_centers, y_centers, c=z_centers)
 plt.gray()
plt.xlabel('Pickup/Dropoff Longitude')
 plt.ylabel('Pickup/Dropoff Latitude')
 plt.title('Cluster Centers of NYC locations')
 plt.show()
regressor = xgb.XGBRegressor(**Best_trial)
 regressor.fit(X_train, y_train, early_stopping_rounds=10, eval_set=[(X_test, y_test)],verbose=False )<predict_on_test>
del X_kmeans, X_df
train_one['pickup_neighborhood'] = kmeans.predict(np.column_stack([train_one.pickup_longitude.values,train_one.pickup_latitude.values]))
train_one['dropoff_neighborhood'] = kmeans.predict(np.column_stack([train_one.dropoff_longitude.values,train_one.dropoff_latitude.values]))
train_one['pickup_neighborhood'] = train_one.pickup_neighborhood.astype(dtype = 'uint8')
 train_one['dropoff_neighborhood'] = train_one.dropoff_neighborhood.astype(dtype = 'uint8')
 train_two['pickup_neighborhood'] = kmeans.predict(np.column_stack([train_two.pickup_longitude.values,train_two.pickup_latitude.values]))
train_two['dropoff_neighborhood'] = kmeans.predict(np.column_stack([train_two.dropoff_longitude.values,train_two.dropoff_latitude.values]))
train_two['pickup_neighborhood'] = train_two.pickup_neighborhood.astype(dtype = 'uint8')
 train_two['dropoff_neighborhood'] = train_two.dropoff_neighborhood.astype(dtype = 'uint8')
 train_three['pickup_neighborhood'] = kmeans.predict(np.column_stack([train_three.pickup_longitude.values,train_three.pickup_latitude.values]))
train_three['dropoff_neighborhood'] = kmeans.predict(np.column_stack([train_three.dropoff_longitude.values,train_three.dropoff_latitude.values]))
train_three['pickup_neighborhood'] = train_three.pickup_neighborhood.astype(dtype = 'uint8')
 train_three['dropoff_neighborhood'] = train_three.dropoff_neighborhood.astype(dtype = 'uint8')
 train_four['pickup_neighborhood'] = kmeans.predict(np.column_stack([train_four.pickup_longitude.values,train_four.pickup_latitude.values]))
train_four['dropoff_neighborhood'] = kmeans.predict(np.column_stack([train_four.dropoff_longitude.values,train_four.dropoff_latitude.values]))
train_four['pickup_neighborhood'] = train_four.pickup_neighborhood.astype(dtype = 'uint8')
 train_four['dropoff_neighborhood'] = train_four.dropoff_neighborhood.astype(dtype = 'uint8')
 test['pickup_neighborhood'] = kmeans.predict(np.column_stack([test.pickup_longitude.values, test.pickup_latitude.values]))
test['dropoff_neighborhood'] = kmeans.predict(np.column_stack([test.dropoff_longitude.values,test.dropoff_latitude.values]))
test['pickup_neighborhood'] = test.pickup_neighborhood.astype(dtype = 'uint8')
 test['dropoff_neighborhood'] = test.dropoff_neighborhood.astype(dtype = 'uint8')
y_xgb_pred = regressor.predict(final_X_test )<save_to_csv>
with open('kmeans_minibatch_200_round4_v2.pkl', 'wb')as fid:
pickle.dump(kmeans, fid)
output1 = pd.DataFrame({"id":test_data.id, "target":y_xgb_pred})
 output1.to_csv('xgb.csv', index=False )<init_hyperparams>
del KMeans, le_dict, MiniBatchKMeans,StandardScaler, calculate_distance, centers, degree_to_radion, holidays, kmeans,nyc_weather, pickle, z, z_centers, x_centers, y_centers
params ={'random_state': 33,'n_estimators':5000,
'min_data_per_group': 5,
'boosting_type': 'gbdt',
'device_type' : 'gpu',
'num_leaves': 256,
'num_iterations' : 5000,
'max_dept': -1,
'learning_rate': 0.005,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'split',
'metric': 'rmse',
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}<train_model>
gc.collect()
dir()
lgb_model = LGBMRegressor(**params)
 lgb_model.fit(X_train, y_train, eval_set=(X_test, y_test),early_stopping_rounds = 50, verbose = 0 )<predict_on_test>
train_full = pd.concat([train_one, train_two, train_three, train_four] )
y_pred_lgb = lgb_model.predict(final_X_test )<save_to_csv>
categorical_cols = ['pickup_neighborhood','dropoff_neighborhood','day_hour','month','year','passenger_count','holiday','hot_day','cold_day','rainy_day','snowy_day','windy_day']
numerical_cols = ['distance']
df_cats = train_full[categorical_cols].copy()
le_dict = {}
for col in categorical_cols:
le_dict[col] = LabelEncoder().fit(df_cats[col])
 df_cats[col] = le_dict[col].transform(df_cats[col])
 X_cats_full = df_cats.values
ohe = OneHotEncoder(categories = 'auto', drop = 'first')
 X_onehot = ohe.fit_transform(X_cats_full)
 X_nums_full = train_full[numerical_cols].values
X_nums_sparse = csr_matrix(X_nums_full)
 X_full = hstack([X_onehot, X_nums_sparse])
 X_full = X_full.tocsr()
si = SimpleImputer()
X = si.fit_transform(X_full)
 y = train_full.fare_amount.values
del X_onehot, X_nums_sparse, X_full, train_full
gc.collect()
output2 = pd.DataFrame({"id":test_data.id, "target":y_pred_lgb})
 output2.to_csv('lgbm.csv', index=False )<init_hyperparams>
df = test.copy()
df_cats = df[categorical_cols].copy()
le_dict = {}
for col in categorical_cols:
le_dict[col] = LabelEncoder().fit(df_cats[col])
 df_cats[col] = le_dict[col].transform(df_cats[col])
 X_cats_full = df_cats.values
X_onehot = ohe.transform(X_cats_full)
 del df_cats, X_cats_full
X_nums_full = df[numerical_cols].values
X_nums_sparse = csr_matrix(X_nums_full)
 del df, X_nums_full
X_full = hstack([X_onehot, X_nums_sparse])
 X_full = X_full.tocsr()
si = SimpleImputer()
X_public = si.fit_transform(X_full)
 del X_onehot, X_nums_sparse, X_full
Best_trial = {'l2_leaf_reg': 0.02247766515106271, 'max_bin': 364, 'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876, 'max_depth': 10, 'random_state': 24, 'min_data_in_leaf': 300,
'loss_function': 'RMSE','n_estimators': 25000,'rsm':0.5}<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size =.1)
cb_model = CatBoostRegressor(**Best_trial )<train_model>
%%time
params = {'objective': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 50,
'max_depth': 8,
'learning_rate': 0.5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 10,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 100}
d_train = lgb.Dataset(X_train, label=y_train)
 d_test = lgb.Dataset(X_test, label=y_test)
 watchlist = [d_train, d_test]
num_rounds = 100
verbose_eval = 100
early_stop = 100
model_lgb = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 pred_test_y_lgb = model_lgb.predict(X_test, num_iteration=model_lgb.best_iteration)
 print("LGB Loss = " + str(sqrt(mean_squared_error(y_test,pred_test_y_lgb))))
cb_model.fit(X_train, y_train,eval_set=(X_test, y_test),use_best_model=True,verbose=1000 )<predict_on_test>
lgb_public= model_lgb.predict(X_public, num_iteration=model_lgb.best_iteration)
 final_pred_public =lgb_public.flatten()
test_predictions_lgb = [float(np.asscalar(x)) for x in final_pred_public]
test_predictions_lgb = [x if x>0 else 0 for x in test_predictions_lgb]
sample = pd.DataFrame({'key': test_id,'fare_amount':test_predictions_lgb})
 sample = sample.reindex(['key', 'fare_amount'], axis=1)
 sample.to_csv('submission_lgb.csv', index=False)
 sample.head()