Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<groupby>
from pandas import read_csv, DataFrame, to_datetime
from numpy import radians, sin, cos, arcsin, sqrt, mean
from sklearn import ensemble
import time
from datetime import datetime
import xgboost as xgb
from sklearn.model_selection import train_test_split
train.groupby('batch')[['signal','open_channels']].agg(['min', 'max', 'median', p5, p95])
<groupby>
train = read_csv(".. /input/train.csv")
 test = read_csv(".. /input/test.csv")
train.groupby('open_channels')[['signal','batch']].agg(['min', 'max', 'median', p5, p95])
<groupby>
train = train[:5000000]
train.groupby(['batch','open_channels'])[['signal']].agg(['min', 'max', 'median', p5, p95])
<statistical_test>
print(train.isnull().sum() )
def maddest(d, axis=None):
return np.mean(np.absolute(d - np.mean(d, axis)) , axis)
 def high_pass_filter(x, low_cutoff=1000, sample_rate=10000):
nyquist = 0.5 * sample_rate
norm_low_cutoff = low_cutoff / nyquist
print(norm_low_cutoff)
 sos = butter(10, Wn=[norm_low_cutoff], btype='highpass', output='sos')
 filtered_sig = signal.sosfilt(sos, x)
 return filtered_sig
def denoise_signal(x, wavelet='db4', level=1):
coeff = pywt.wavedec(x, wavelet, mode="per")
 sigma =(1/0.6745)* maddest(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(x)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per' )<feature_engineering>
train = train.dropna(how = 'any', axis = 'rows' )
train = add_rooling_data(train )<load_from_csv>
pickup_longitude_min = test.pickup_longitude.min()
pickup_longitude_max = test.pickup_latitude.max()
pickup_latitude_min = test.pickup_latitude.min()
pickup_latitude_max = test.pickup_latitude.max()
dropoff_longitude_min = test.dropoff_longitude.min()
dropoff_longitude_max = test.dropoff_longitude.max()
dropoff_latitude_min = test.dropoff_latitude.min()
dropoff_latitude_max = test.dropoff_latitude.max()
train = train.loc[(train['fare_amount'] > 0)&(train['fare_amount'] < 300)]
train = train.loc[(train['pickup_longitude'] > pickup_longitude_min)&(train['pickup_longitude'] < pickup_longitude_max)]
train = train.loc[(train['pickup_latitude'] > pickup_latitude_min)&(train['pickup_latitude'] < pickup_latitude_max)]
train = train.loc[(train['dropoff_longitude'] > dropoff_longitude_min)&(train['dropoff_longitude'] < dropoff_longitude_max)]
train = train.loc[(train['dropoff_latitude'] > dropoff_latitude_min)&(train['dropoff_latitude'] < dropoff_latitude_max)]
train = pd.read_csv(os.path.join(base + '/train.csv'))
test = pd.read_csv(os.path.join(base + '/test.csv'))
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal']
return df
train = features(train)
 test = features(test)
 col = [c for c in train.columns if c not in ['time', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]
x1, x2, y1, y2 = model_selection.train_test_split(train[col], train['open_channels'], test_size=0.3, random_state=7)
 target = train['open_channels']
train = train[col]
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = metrics.f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
<train_model>
def rasst(value1, value2, value3, value4):
longitude_1, latitude_1, longitude_2, latitude_2 = value1, value2, value3, value4
longitude_1, latitude_1, longitude_2, latitude_2 = map(radians, [longitude_1, latitude_1, longitude_2, latitude_2])
 dlongitude = longitude_2 - longitude_1
dlatitude = latitude_2 - latitude_1
value = sin(dlatitude/2.0)**2 + cos(latitude_1)* cos(latitude_2)* sin(dlongitude/2.0)**2
c = 2 * arcsin(sqrt(value))
km = c * 6367
return km
params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=0, early_stopping_rounds=250, feval=MacroF1Metric)
 preds_lgb =(model.predict(test[col], num_iteration=model.best_iteration)).astype(np.float16)
 oof_lgb =(model.predict(train, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()<train_model>
train['pickup_datetime'] = to_datetime(train['pickup_datetime'])
 train['hour_of_day'] = train.pickup_datetime.dt.hour.astype(float)
 train['day'] = train.pickup_datetime.dt.day.astype(float)
 train['week'] = train.pickup_datetime.dt.week.astype(float)
 train['month'] = train.pickup_datetime.dt.month.astype(float)
 train['day_of_year'] = train.pickup_datetime.dt.dayofyear.astype(float)
 train['week_of_year'] = train.pickup_datetime.dt.weekofyear.astype(float)
 train['passenger_count'] = train['passenger_count'].astype(float)
 train['rasst'] = rasst(train['pickup_longitude'], train['pickup_latitude'], train['dropoff_longitude'], train['dropoff_latitude'])
 test['pickup_datetime'] = to_datetime(test['pickup_datetime'])
 test['hour_of_day'] = test.pickup_datetime.dt.hour.astype(float)
 test['day'] = test.pickup_datetime.dt.day.astype(float)
 test['week'] = test.pickup_datetime.dt.week.astype(float)
 test['month'] = test.pickup_datetime.dt.month.astype(float)
 test['day_of_year'] = test.pickup_datetime.dt.dayofyear.astype(float)
 test['week_of_year'] = test.pickup_datetime.dt.weekofyear.astype(float)
 test['passenger_count'] = test['passenger_count'].astype(float)
 test['rasst'] = rasst(test['pickup_longitude'], test['pickup_latitude'], test['dropoff_longitude'], test['dropoff_latitude'] )
params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=0, early_stopping_rounds=250)
 preds_xgb = model.predict(xgb.DMatrix(test[col]), ntree_limit=model.best_ntree_limit)
 oof_xgb =(model.predict(xgb.DMatrix(train), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 del train_set, val_set; gc.collect()<train_model>
train_y = train["fare_amount"]
test_key = test['key']
train_x = train.drop(["fare_amount", "key"], axis = 1)
 train_x = train_x.drop(['pickup_datetime'], axis = 1)
 test = test.drop(['pickup_datetime', 'key'], axis = 1)
model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=0, early_stopping_rounds=250)
 preds_cb =(model.predict(test[col])).astype(np.float16)
 oof_cb =(model.predict(train)).astype(np.float16)
 del train_dataset, eval_dataset, model; gc.collect()<define_variables>
train_xgb = xgb.DMatrix(train_x, train_y)
 test_xgb = xgb.DMatrix(test)
final_preds = 0.4 * preds_lgb + 0.4 * preds_xgb + 0.2 * preds_cb
final_oof = 0.4 * oof_lgb + 0.4 * oof_xgb + 0.2 * oof_cb<feature_engineering>
num_round = 10
param = {'max_depth':20, 'eta':0.5,'min_child-weight':2, 'gamma':2, 'booster':'dart', 'three-method':'approx', 'normalize_type':'forest', 'rate_drop':0.5, 'eval_metric':'rmse'}
train = xgb.train(param, train_xgb, num_round)
 predict = train.predict(test_xgb, ntree_limit = num_round )
def optimize_predictions(preds, coeffs):
preds[preds <= coeffs[0]] = 0
preds[np.where(np.logical_and(preds > coeffs[0], preds <= coeffs[1])) ] = 1
preds[np.where(np.logical_and(preds > coeffs[1], preds <= coeffs[2])) ] = 2
preds[np.where(np.logical_and(preds > coeffs[2], preds <= coeffs[3])) ] = 3
preds[np.where(np.logical_and(preds > coeffs[3], preds <= coeffs[4])) ] = 4
preds[np.where(np.logical_and(preds > coeffs[4], preds <= coeffs[5])) ] = 5
preds[np.where(np.logical_and(preds > coeffs[5], preds <= coeffs[6])) ] = 6
preds[np.where(np.logical_and(preds > coeffs[6], preds <= coeffs[7])) ] = 7
preds[np.where(np.logical_and(preds > coeffs[7], preds <= coeffs[8])) ] = 8
preds[np.where(np.logical_and(preds > coeffs[8], preds <= coeffs[9])) ] = 9
preds[preds > coeffs[9]] = 10
preds = preds.astype(np.int8)
 return preds<save_to_csv>
submission = DataFrame({
"key": test_key,
"fare_amount": predict.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()