Kaggle competition: New York City Taxi Fare Prediction
df_train = pd.read_csv(".. /input/liverpool-ion-switching/train.csv")
 train_time = df_train["time"].values.reshape(-1,500000)
 train_signal = df_train["signal"].values.reshape(-1,500000)
 train_opench = df_train["open_channels"].values.reshape(-1,500000 )<statistical_test>
plt.figure(figsize=(8, 5), dpi=80)
 sns.set_style("darkgrid" )
def markov_p(data):
channel_range = np.unique(data)
 channel_bins = np.append(channel_range, 11)
 data_next = np.roll(data, -1)
 matrix = []
for i in channel_range:
current_row = np.histogram(data_next[data == i], bins=channel_bins)[0]
current_row = current_row / np.sum(current_row)
 matrix.append(current_row)
 return np.array(matrix)
 p03 = markov_p(train_opench[3])
 p04 = markov_p(train_opench[4])
 eig_values, eig_vectors = np.linalg.eig(np.transpose(p03))
print("Eigenvalues :", eig_values )<define_variables>
test_dataset = pd.read_csv('.. /input/test.csv')
 train_dataset = pd.read_csv('.. /input/train.csv', nrows=2_000_000 )
prob = [.3,.3,.2,.1,.05,.05]
data = train_signal[4][1:7]
len(data)
 stat = []
for i in range(1, 1000):
mc_signal = np.random.choice(data, 500, p=prob)
 df = pd.DataFrame(index=range(500), data={'mc_signal': mc_signal,})
 stat.append([df['mc_signal'].sum().round(0)] )<set_options>
train_dataset.drop(labels='key', axis=1, inplace=True)
 test_dataset.drop(labels='key', axis=1, inplace=True )
ks(arsig, 'norm' )<set_options>
train_dataset.passenger_count = train_dataset.passenger_count.astype(dtype = 'uint8' )
ks(p03, 'norm' )<set_options>
train_dataset.pickup_longitude = train_dataset.pickup_longitude.astype(dtype = 'float32')
 train_dataset.pickup_latitude = train_dataset.pickup_latitude.astype(dtype = 'float32')
 train_dataset.dropoff_longitude = train_dataset.dropoff_longitude.astype(dtype = 'float32')
 train_dataset.dropoff_latitude = train_dataset.dropoff_latitude.astype(dtype = 'float32')
 train_dataset.fare_amount = train_dataset.fare_amount.astype(dtype = 'float32' )
cf.go_offline()
py.init_notebook_mode()
cf.getThemes()
cf.set_config_file(theme='ggplot')
 warnings.simplefilter('ignore')
 pd.plotting.register_matplotlib_converters()
sns.mpl.rc('figure',figsize=(16, 6))
plt.style.use('ggplot')
 sns.set_style('darkgrid')
<load_from_csv>
train_dataset.isnull().sum()
base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train = pd.read_csv(os.path.join(base + '/train.csv'))
test = pd.read_csv(os.path.join(base + '/test.csv'))<feature_engineering>
train_dataset['pickup_datetime'] = pd.to_datetime(arg=train_dataset['pickup_datetime'], infer_datetime_format=True)
 test_dataset['pickup_datetime'] = pd.to_datetime(arg=test_dataset['pickup_datetime'], infer_datetime_format=True )
train2 = train.copy()
a=500000; b=600000
train2.loc[train.index[a:b],'signal2'] = train2.signal[a:b].values - 3*(train2.time.values[a:b] - 50)/10.
 def f(x,low,high,mid): return -(( -low+high)/625)*(x-mid)**2+high -low
batch = 7; a = 500000*(batch-1); b = 500000*batch
train2.loc[train2.index[a:b],'signal2'] = train.signal.values[a:b] - f(train.time[a:b].values,-1.817,3.186,325)
 batch = 8; a = 500000*(batch-1); b = 500000*batch
train2.loc[train2.index[a:b],'signal2'] = train.signal.values[a:b] - f(train.time[a:b].values,-0.094,4.936,375)
 batch = 9; a = 500000*(batch-1); b = 500000*batch
train2.loc[train2.index[a:b],'signal2'] = train.signal.values[a:b] - f(train.time[a:b].values,1.715,6.689,425)
 batch = 10; a = 500000*(batch-1); b = 500000*batch
train2.loc[train2.index[a:b],'signal2'] = train.signal.values[a:b] - f(train.time[a:b].values,3.361,8.45,475 )<feature_engineering>
def add_new_date_time_features(dataset):
dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['year'] = dataset.pickup_datetime.dt.year
dataset['day_of_week'] = dataset.pickup_datetime.dt.dayofweek
return dataset
train_dataset = add_new_date_time_features(train_dataset)
 test_dataset = add_new_date_time_features(test_dataset )
test2 = test.copy()
start=500
a = 0; b = 100000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 start=510
a = 100000; b = 200000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 start=540
a = 400000; b = 500000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 start=560
a = 600000; b = 700000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 start=570
a = 700000; b = 800000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 start=580
a = 800000; b = 900000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - 3*(test2.time.values[a:b]-start)/10.
 def f(x):
return -(0.00788)*(x-625)**2+2.345 +2.58
a = 1000000; b = 1500000
test2.loc[test2.index[a:b],'signal2'] = test2.signal.values[a:b] - f(test2.time[a:b].values )<feature_engineering>
train_dataset.fare_amount[(train_dataset.fare_amount <= 0)|(train_dataset.fare_amount >= 350)].count()
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for j in ['signal']:
for c in ['batch','batch_slices2']:
d = {}
d['mean'+c+j] = df.groupby([c])[j].mean()
d['median'+c+j] = df.groupby([c])[j].median()
d['max'+c+j] = df.groupby([c])[j].max()
d['min'+c+j] = df.groupby([c])[j].min()
d['std'+c+j] = df.groupby([c])[j].std()
d['mean_abs_chg'+c+j] = df.groupby([c])[j].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c+j] = df.groupby([c])[j].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c+j] = df.groupby([c])[j].apply(lambda x: np.min(np.abs(x)))
 d['range'+c+j] = d['max'+c+j] - d['min'+c+j]
d['maxtomin'+c+j] = d['max'+c+j] / d['min'+c+j]
d['abs_avg'+c+j] =(d['abs_min'+c+j] + d['abs_max'+c+j])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'+j] = df[j].shift(1)
 df['signal_shift_-1'+j] = df[j].shift(-1)
 for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal']
return df
train2 = features(train2)
 test = features(test2 )<define_variables>
print(f'Row count before elimination - {train_dataset.shape[0]}')
 train_dataset = train_dataset[train_dataset.fare_amount.between(0, 350, inclusive=False)]
print(f'Row count after elimination - {train_dataset.shape[0]}' )
col = [c for c in train2.columns if c not in ['time', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]<compute_test_metric>
train_dataset.passenger_count[(train_dataset.passenger_count < 1)|(train_dataset.passenger_count > 8)].count()
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = metrics.f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )<define_variables>
print(f'Row count before elimination - {train_dataset.shape[0]}')
 train_dataset = train_dataset[train_dataset.passenger_count.between(0, 8, inclusive=False)]
print(f'Row count after elimination - {train_dataset.shape[0]}' )
model_batch = [[1,2],[3,7],[4,8],[6,9],[5,10]]<load_from_csv>
def degree_to_radion(degree):
return degree*(np.pi/180)
 def calculate_distance(pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude):
from_lat = degree_to_radion(pickup_latitude)
 from_long = degree_to_radion(pickup_longitude)
 to_lat = degree_to_radion(dropoff_latitude)
 to_long = degree_to_radion(dropoff_longitude)
 radius = 6371.01
lat_diff = to_lat - from_lat
long_diff = to_long - from_long
a = np.sin(lat_diff / 2)**2 + np.cos(degree_to_radion(from_lat)) * np.cos(degree_to_radion(to_lat)) * np.sin(long_diff / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
return radius * c
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv' )<define_variables>
train_dataset['distance'] = calculate_distance(train_dataset.pickup_latitude, train_dataset.pickup_longitude, train_dataset.dropoff_latitude, train_dataset.dropoff_longitude)
 test_dataset['distance'] = calculate_distance(test_dataset.pickup_latitude, test_dataset.pickup_longitude, test_dataset.dropoff_latitude, test_dataset.dropoff_longitude )
valid_predict_LGBM =[]
valid_target_LGBM =[]
valid_predict_XGB =[]
valid_target_XGB =[]
valid_predict_CAT =[]
valid_target_CAT =[]<prepare_output>
train_dataset.sort_values(by='distance' )
sub['LGBM_preds']=sub.open_channels
sub['XGB_preds']=sub.open_channels
sub['CAT_preds']=sub.open_channels<prepare_x_and_y>
train_dataset.distance[(train_dataset.distance == 0)].count()
for j,idx in enumerate(model_batch[0]):
if j ==0:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split = train2[col].loc[train2.index[a:b]]
train_model_target = train2.loc[train2.index[a:b],'open_channels']
else:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split1 = train2[col].loc[train2.index[a:b]]
train_model_target1 = train2.loc[train2.index[a:b],'open_channels']
train_model_split = np.concatenate(( train_model_split,train_model_split1),axis = 0)
 train_model_target = np.concatenate(( train_model_target,train_model_target1),axis = 0)
 x1, x2, y1, y2 = model_selection.train_test_split(train_model_split, train_model_target , test_size=0.2, random_state=49)
 print('============================ start training LGBM ==================================')
 params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=250, feval=MacroF1Metric)
 oof_valid_lgb =(model.predict(x2, num_iteration=model.best_iteration)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_lgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
valid_predict_LGBM = valid_predict_LGBM + oof_valid_lgb.tolist()
valid_target_LGBM = valid_target_LGBM + np.array(y2 ).tolist()
print('============================ start LGBM predict==================================')
 a = 0
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 a = 3
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 a = 8
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 test_batch = test[col].loc[test.index[1000000:2000000]]
print('start test last subsample'+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[1000000:2000000,2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 print(gc.collect())
 print('============================ start training XGBBOOST ==================================')
 params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=100, early_stopping_rounds=250)
 oof_valid_xgb =(model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_xgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_set, val_set; gc.collect()
valid_predict_XGB = valid_predict_XGB + oof_valid_xgb.tolist()
valid_target_XGB = valid_target_XGB + np.array(y2 ).tolist()
print('============================ start XGB predict==================================')
 a = 0
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 a = 3
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 a = 8
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 test_batch = test[col].loc[test.index[1000000:2000000]]
print('start test last subsample'+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[1000000:2000000,3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print('============================ start training Catboost ==================================')
 model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=100, early_stopping_rounds=250)
 oof_valid_cb =(model.predict(x2)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_cb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_dataset, eval_dataset
print(gc.collect())
 valid_predict_CAT = valid_predict_CAT + oof_valid_cb.tolist()
valid_target_CAT = valid_target_CAT + np.array(y2 ).tolist()
print('============================ start Catboost predict==================================')
 a = 0
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 a = 3
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 a = 8
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 test_batch = test[col].loc[test.index[1000000:2000000]]
print('start test last subsample'+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[1000000:2000000,4] =(model.predict(test_batch)).astype(np.float16 )<prepare_x_and_y>
train_dataset[(train_dataset.pickup_latitude != train_dataset.dropoff_latitude)& 
(train_dataset.pickup_longitude != train_dataset.dropoff_latitude)& 
(train_dataset.distance == 0)].count()
for j,idx in enumerate(model_batch[1]):
if j ==0:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split = train2[col].loc[train2.index[a:b]]
train_model_target = train2.loc[train2.index[a:b],'open_channels']
else:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split1 = train2[col].loc[train2.index[a:b]]
train_model_target1 = train2.loc[train2.index[a:b],'open_channels']
train_model_split = np.concatenate(( train_model_split,train_model_split1),axis = 0)
 train_model_target = np.concatenate(( train_model_target,train_model_target1),axis = 0)
 x1, x2, y1, y2 = model_selection.train_test_split(train_model_split, train_model_target , test_size=0.2, random_state=49)
 print('============================ start training LGBM ==================================')
 params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=250, feval=MacroF1Metric)
 oof_valid_lgb =(model.predict(x2, num_iteration=model.best_iteration)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_lgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
valid_predict_LGBM = valid_predict_LGBM + oof_valid_lgb.tolist()
valid_target_LGBM = valid_target_LGBM + np.array(y2 ).tolist()
print('============================ start LGBM predict==================================')
 a = 4
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()
print('============================ start training XGBBOOST ==================================')
 params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=100, early_stopping_rounds=250)
 oof_valid_xgb =(model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_xgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_set, val_set; gc.collect()
valid_predict_XGB = valid_predict_XGB + oof_valid_xgb.tolist()
valid_target_XGB = valid_target_XGB + np.array(y2 ).tolist()
print('============================ start XGB predict==================================')
 a = 4
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print('============================ start training Catboost ==================================')
 model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=100, early_stopping_rounds=250)
 oof_valid_cb =(model.predict(x2)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_cb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_dataset, eval_dataset; gc.collect()
valid_predict_CAT = valid_predict_CAT + oof_valid_cb.tolist()
valid_target_CAT = valid_target_CAT + np.array(y2 ).tolist()
print('============================ start Catboost predict==================================')
 a = 4
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
<prepare_x_and_y>
def add_distances_from_airport(dataset):
jfk_coords =(40.639722, -73.778889)
 ewr_coords =(40.6925, -74.168611)
 lga_coords =(40.77725, -73.872611)
 dataset['pickup_jfk_distance'] = calculate_distance(jfk_coords[0], jfk_coords[1], dataset.pickup_latitude, dataset.pickup_longitude)
 dataset['dropof_jfk_distance'] = calculate_distance(jfk_coords[0], jfk_coords[1], dataset.dropoff_latitude, dataset.dropoff_longitude)
 dataset['pickup_ewr_distance'] = calculate_distance(ewr_coords[0], ewr_coords[1], dataset.pickup_latitude, dataset.pickup_longitude)
 dataset['dropof_ewr_distance'] = calculate_distance(ewr_coords[0], ewr_coords[1], dataset.dropoff_latitude, dataset.dropoff_longitude)
 dataset['pickup_lga_distance'] = calculate_distance(lga_coords[0], lga_coords[1], dataset.pickup_latitude, dataset.pickup_longitude)
 dataset['dropof_lga_distance'] = calculate_distance(lga_coords[0], lga_coords[1], dataset.dropoff_latitude, dataset.dropoff_longitude)
 return dataset
train_dataset = add_distances_from_airport(train_dataset)
 test_dataset = add_distances_from_airport(test_dataset )
for j,idx in enumerate(model_batch[2]):
if j ==0:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split = train2[col].loc[train2.index[a:b]]
train_model_target = train2.loc[train2.index[a:b],'open_channels']
else:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split1 = train2[col].loc[train2.index[a:b]]
train_model_target1 = train2.loc[train2.index[a:b],'open_channels']
train_model_split = np.concatenate(( train_model_split,train_model_split1),axis = 0)
 train_model_target = np.concatenate(( train_model_target,train_model_target1),axis = 0)
 x1, x2, y1, y2 = model_selection.train_test_split(train_model_split, train_model_target , test_size=0.2, random_state=49)
 print('============================ start training LGBM ==================================')
 params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=250, feval=MacroF1Metric)
 oof_valid_lgb =(model.predict(x2, num_iteration=model.best_iteration)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_lgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
valid_predict_LGBM = valid_predict_LGBM + oof_valid_lgb.tolist()
valid_target_LGBM = valid_target_LGBM + np.array(y2 ).tolist()
print('============================ start LGBM predict==================================')
 a = 1
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 a = 9
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()
print('============================ start training XGBBOOST ==================================')
 params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=100, early_stopping_rounds=250)
 oof_valid_xgb =(model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_xgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_set, val_set; gc.collect()
valid_predict_XGB = valid_predict_XGB + oof_valid_xgb.tolist()
valid_target_XGB = valid_target_XGB + np.array(y2 ).tolist()
print('============================ start XGB predict==================================')
 a = 1
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 a = 9
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print('============================ start training Catboost ==================================')
 model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=100, early_stopping_rounds=250)
 oof_valid_cb =(model.predict(x2)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_cb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_dataset, eval_dataset; gc.collect()
valid_predict_CAT = valid_predict_CAT + oof_valid_cb.tolist()
valid_target_CAT = valid_target_CAT + np.array(y2 ).tolist()
print('============================ start Catboost predict==================================')
 a = 1
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 a = 9
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16 )<prepare_x_and_y>
train_dataset[(train_dataset.distance>90)&(train_dataset.fare_amount<70)]
for j,idx in enumerate(model_batch[3]):
if j ==0:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split = train2[col].loc[train2.index[a:b]]
train_model_target = train2.loc[train2.index[a:b],'open_channels']
else:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split1 = train2[col].loc[train2.index[a:b]]
train_model_target1 = train2.loc[train2.index[a:b],'open_channels']
train_model_split = np.concatenate(( train_model_split,train_model_split1),axis = 0)
 train_model_target = np.concatenate(( train_model_target,train_model_target1),axis = 0)
 x1, x2, y1, y2 = model_selection.train_test_split(train_model_split, train_model_target , test_size=0.2, random_state=49)
 print('============================ start training LGBM ==================================')
 params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=250, feval=MacroF1Metric)
 oof_valid_lgb =(model.predict(x2, num_iteration=model.best_iteration)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_lgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
valid_predict_LGBM = valid_predict_LGBM + oof_valid_lgb.tolist()
valid_target_LGBM = valid_target_LGBM + np.array(y2 ).tolist()
print('============================ start LGBM predict==================================')
 a = 2
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 a = 6
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()
print('============================ start training XGBBOOST ==================================')
 params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=100, early_stopping_rounds=250)
 oof_valid_xgb =(model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_xgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_set, val_set; gc.collect()
valid_predict_XGB = valid_predict_XGB + oof_valid_xgb.tolist()
valid_target_XGB = valid_target_XGB + np.array(y2 ).tolist()
print('============================ start XGB predict==================================')
 a = 2
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 a = 6
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print('============================ start training Catboost ==================================')
 model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=100, early_stopping_rounds=250)
 oof_valid_cb =(model.predict(x2)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_cb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_dataset, eval_dataset; gc.collect()
valid_predict_CAT = valid_predict_CAT + oof_valid_cb.tolist()
valid_target_CAT = valid_target_CAT + np.array(y2 ).tolist()
print('============================ start Catboost predict==================================')
 a = 2
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 a = 6
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16 )<prepare_x_and_y>
selected_predictors = [
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'pickup_jfk_distance',
'dropof_jfk_distance',
'pickup_ewr_distance',
'dropof_ewr_distance',
'pickup_lga_distance',
'dropof_lga_distance',
'hour',
'month',
'year',
'distance'
]
X = train_dataset.loc[:, selected_predictors].values
y = train_dataset.iloc[:, 0].values
X_test_dataset = test_dataset.loc[:, selected_predictors].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/20 )
for j,idx in enumerate(model_batch[4]):
if j ==0:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split = train2[col].loc[train2.index[a:b]]
train_model_target = train2.loc[train2.index[a:b],'open_channels']
else:
batch = idx; a = 500000*(batch-1); b = 500000*batch
train_model_split1 = train2[col].loc[train2.index[a:b]]
train_model_target1 = train2.loc[train2.index[a:b],'open_channels']
train_model_split = np.concatenate(( train_model_split,train_model_split1),axis = 0)
 train_model_target = np.concatenate(( train_model_target,train_model_target1),axis = 0)
 x1, x2, y1, y2 = model_selection.train_test_split(train_model_split, train_model_target , test_size=0.2, random_state=49)
 print('============================ start training LGBM ==================================')
 params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=250, feval=MacroF1Metric)
 oof_valid_lgb =(model.predict(x2, num_iteration=model.best_iteration)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_lgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
valid_predict_LGBM = valid_predict_LGBM + oof_valid_lgb.tolist()
valid_target_LGBM = valid_target_LGBM + np.array(y2 ).tolist()
print('============================ start LGBM predict==================================')
 a = 5
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 a = 7
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),2] =(model.predict(test_batch, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()
print('============================ start training XGBBOOST ==================================')
 params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=100, early_stopping_rounds=250)
 oof_valid_xgb =(model.predict(xgb.DMatrix(x2), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_xgb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_set, val_set; gc.collect()
valid_predict_XGB = valid_predict_XGB + oof_valid_xgb.tolist()
valid_target_XGB = valid_target_XGB + np.array(y2 ).tolist()
print('============================ start XGB predict==================================')
 a = 5
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 a = 7
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),3] =(model.predict(xgb.DMatrix(np.array(test_batch)) , ntree_limit=model.best_ntree_limit)).astype(np.float16)
 print('============================ start training Catboost ==================================')
 model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=100, early_stopping_rounds=250)
 oof_valid_cb =(model.predict(x2)).astype(np.float16)
 print("BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(oof_valid_cb, 0, 10)).astype(int),np.array(y2),average = 'macro'))
del train_dataset, eval_dataset; gc.collect()
valid_predict_CAT = valid_predict_CAT + oof_valid_cb.tolist()
valid_target_CAT = valid_target_CAT + np.array(y2 ).tolist()
print('============================ start Catboost predict==================================')
 a = 5
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
 a = 7
test_batch = test[col].loc[test.index[100000*a:100000*(a+1)]]
print('start test subsample {}'.format(a)+', test_batch shape= ' + str(test_batch.shape))
sub.iloc[100000*a:100000*(a+1),4] =(model.predict(test_batch)).astype(np.float16)
<compute_test_metric>
rand_forest_regressor = RandomForestRegressor()
rand_forest_regressor.fit(X_train, y_train)
 y_rand_forest_predict = rand_forest_regressor.predict(X_test)
 random_forest_model_error = sqrt(mean_squared_error(np.expm1(y_test), np.expm1(y_rand_forest_predict)))
 print(f' Random Forest Mean Squared Error - {random_forest_model_error}' )
print("LGBM BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(valid_predict_LGBM, 0, 10)).astype(int),np.array(valid_target_LGBM),average = 'macro'))
print("XGB BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(valid_predict_XGB, 0, 10)).astype(int),np.array(valid_target_XGB),average = 'macro'))
print("XGB BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(valid_predict_CAT, 0, 10)).astype(int),np.array(valid_target_CAT),average = 'macro'))<compute_test_metric>
Ensemble_predict =np.array(valid_predict_LGBM)*0.4+np.array(valid_predict_XGB)*0.4+np.array(valid_predict_CAT)*0.2
print("Ensemble BEST VALIDATION_SCORE(F1): ", f1_score(np.round(np.clip(Ensemble_predict, 0, 10)).astype(int),np.array(valid_target_XGB),average = 'macro'))<train_model>
XGB_model = XGBRegressor(learning_rate=0.3, max_depth=6, n_estimators=500)
 XGB_model.fit(X_train, y_train)
 y_XGB_predict = XGB_model.predict(X_test)
 XGB_model_error = sqrt(mean_squared_error(np.expm1(y_test), np.expm1(y_XGB_predict)))
 print(f'XGBoost Mean Squared Error - {XGB_model_error}' )
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
 return -f1_score(y, X_p,average = 'macro')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
 def coefficients(self):
return self.coef_['x']<train_on_grid>
lgb_model = lgb.LGBMRegressor(objective='regression',num_leaves=35, n_estimators=300)
 lgb_model.fit(X_train, y_train)
 y_LGB_predict = lgb_model.predict(X_test)
 LGB_model_error = sqrt(mean_squared_error(np.expm1(y_test), np.expm1(y_LGB_predict)))
 print(f'LGBM Mean Squared Error - {LGB_model_error}' )
%%time
optR = OptimizedRounder()
optR.fit(np.array(Ensemble_predict ).reshape(-1,), np.array(valid_target_XGB))
coefficients = optR.coefficients()
print(coefficients )<predict_on_test>
ensembled_prediction =(0.5*np.expm1(y_XGB_predict)) +(0.5*np.expm1(y_LGB_predict))
ensembled_prediction_error = sqrt(mean_squared_error(np.expm1(y_test), ensembled_prediction))
print(f'Ensembled Mean Squared Error - {ensembled_prediction_error}' )
opt_preds = optR.predict(np.array(Ensemble_predict ).reshape(-1,), coefficients)
 print('f1', metrics.f1_score(np.array(valid_target_XGB), opt_preds, average = 'macro'))<predict_on_test>
y_XGB_predict = np.expm1(XGB_model.predict(X_test_dataset))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = y_XGB_predict
submission.to_csv('xgb_submission.csv', index=False)
 submission.head(10 )
predict_logic = sub.LGBM_preds*0.4+sub.XGB_preds*0.4+sub.CAT_preds*0.2
opt_test_preds = optR.predict(np.array(predict_logic ).reshape(-1,), coefficients )<save_to_csv>
y_LGB_predict = np.expm1(lgb_model.predict(X_test_dataset))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = y_LGB_predict
submission.to_csv('lgbm_submission.csv', index=False)
 submission.head(10 )
submit = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 submit.open_channels = opt_test_preds.astype(int)
 submit.to_csv('submission_lgb.csv', index=False, float_format='%.4f' )<data_type_conversions>
ensembled_prediction =(0.5*y_XGB_predict)+(0.5*y_LGB_predict)
 submission.to_csv('ensembled_submission.csv', index=False)
 submission.head(10 )