Kaggle competition: New York City Taxi Fare Prediction
%%time
X_test = create_X(test_df, target_meter=3)
 gc.collect()
y_test3 = pred(X_test, models3)
 del X_test
gc.collect()<load_from_disk>
traintypes = {'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
cols = list(traintypes.keys() )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )<define_variables>
%%time
train = pd.read_csv(TRAIN_PATH, usecols=cols, dtype=traintypes, nrows = 5000000 )
print(np.sum(y_test0 < 0))
print(np.sum(y_test1 < 0))
print(np.sum(y_test2 < 0))
print(np.sum(y_test3 < 0))
y_test0 = np.where(y_test0 < 0, 0, y_test0)
 y_test1 = np.where(y_test1 < 0, 0, y_test1)
 y_test2 = np.where(y_test2 < 0, 0, y_test2)
 y_test3 = np.where(y_test3 < 0, 0, y_test3 )<feature_engineering>
print("old size: %d" % len(train))
train = train[train.fare_amount >=0]
print("New size: %d" % len(train))
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<save_to_csv>
train.isnull().sum()
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<set_options>
print("old size: %d" % len(train))
train = train.dropna(how='any', axis=0)
 print("New size after dropping missing value: %d" % len(train))
%matplotlib inline 
<load_from_csv>
% matplotlib inline
plt.style.use('seaborn-whitegrid' )
data = pd.read_csv('.. /input/clean-2apply-kalman/train_clean_2apply_kalman.csv')
 data.head()<choose_model_class>
test = pd.read_csv(".. /input/test.csv")
 print("shape of test data", test.shape)
 test.head()
<create_dataframe>
def select_within_test_boundary(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3] )
<string_transform>
def prepare_time_features(df):
df['pickup_datetime'] = df['pickup_datetime'].str.slice(0, 16)
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], utc=True, format='%Y-%m-%d %H:%M')
 df['hour_of_day'] = df.pickup_datetime.dt.hour
df['month'] = df.pickup_datetime.dt.month
df["year"] = df.pickup_datetime.dt.year
df["weekday"] = df.pickup_datetime.dt.weekday
return df
df = divide_and_add_features(data['signal'])
 df.head()<split>
train = prepare_time_features(train)
 test = prepare_time_features(test )
x_train, x_test, y_train, y_test = train_test_split(df.values, data['open_channels'].values, test_size=0.2)
 del data, df
print('x_train.shape=', x_train.shape)
 print('x_test.shape=', x_test.shape)
 print('y_train.shape=', y_train.shape)
 print('y_test.shape=', y_test.shape )<choose_model_class>
def distance(lat1, lon1, lat2, lon2):
p = 0.017453292519943295
a = 0.5 - np.cos(( lat2 - lat1)* p)/2 + np.cos(lat1 * p)* np.cos(lat2 * p)*(1 - np.cos(( lon2 - lon1)* p)) / 2
return 0.6213712 * 12742 * np.arcsin(np.sqrt(a))
def create_mpl(shape):

 X_input = L.Input(shape)
 X = L.Dense(150, activation='relu' )(X_input)
 X = L.Dense(150, activation='relu' )(X)
 X = L.Dense(125, activation='relu' )(X)
 X = L.Dense(100, activation='relu' )(X)
 X = L.Dense(75, activation='relu' )(X)
 X = L.Dense(50, activation='relu' )(X)
 X = L.Dense(25, activation='relu' )(X)
 X = L.Dense(11, activation='softmax' )(X)
 model = Model(inputs=X_input, outputs=X)
 return model
mlp = create_mpl(x_train[0].shape)
 mlp.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 print(mlp.summary() )<train_model>
train['distance_miles'] = distance(train.pickup_latitude, train.pickup_longitude, \
train.dropoff_latitude, train.dropoff_longitude )
mlp.fit(x=x_train, y=y_train, epochs=30, batch_size=1024, class_weight=class_weight )<predict_on_test>
test['distance_miles'] = distance(test.pickup_latitude, test.pickup_longitude, \
test.dropoff_latitude, test.dropoff_longitude )
mlp_pred = mlp.predict(x_test)
 print('mlp_pred.shape=', mlp_pred.shape )<set_options>
def transform(data):
jfk =(-73.7781, 40.6413)
 ewr =(-74.1745, 40.6895)
 lgr =(-73.8740, 40.7769)
 data['pickup_distance_to_jfk'] = distance(jfk[1], jfk[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_jfk'] = distance(jfk[1], jfk[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_ewr'] = distance(ewr[1], ewr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_ewr'] = distance(ewr[1], ewr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_lgr'] = distance(lgr[1], lgr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_lgr'] = distance(lgr[1], lgr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 return data
train = transform(train)
 test = transform(test )
!nvidia-smi<create_dataframe>
print("old size: %d" % len(train))
train = train.drop(index= train[train['fare_amount']==0].index, axis=0)
 print("New size: %d" % len(train))
dataset = lgb.Dataset(x_train, label=y_train)
 params = {'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'num_leaves': 280,
'learning_rate': 0.026623466966581126,
'max_depth': 73,
'lambda_l1': 2.959759088169741,
'lambda_l2': 1.331172832164913,
'bagging_fraction': 0.9655406551472153,
'bagging_freq': 9,
'colsample_bytree': 0.6867118652742716,
}<set_options>
print("old size: %d" % len(train))
train = train.drop(index= train[train['fare_amount'] < 2.5].index, axis=0)
 print("New size: %d" % len(train))
gc.collect()<train_model>
print("old size: %d" % len(train))
train = train.drop(index= train[train.passenger_count >= 7].index, axis=0)
 print("New size: %d" % len(train))
print('Training LGBM...')
 gbc = lgb.train(params, dataset,
num_boost_round=10000,
verbose_eval=100)
 print('LGBM trained!' )<predict_on_test>
pd.cut(test['distance_miles'],np.linspace(0, 70, num = 8)).value_counts()
gbc_pred = gbc.predict(x_test)
 print('gbc_pred.shape=', gbc_pred.shape )<categorify>
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('fare_amount', axis=1),
df_train['fare_amount'], test_size=0.2, random_state = 42)
 print(X_train.shape)
 print(X_test.shape)
 print(y_train.shape)
 print(y_test.shape )
def convert_to_one_hot(pred):

 pred = np.clip(pred, 0, 10)
 pred = 1 - np.abs(pred.reshape(( -1,1)) - np.arange(11))
pred = np.clip(pred, 0, 1)
 return pred
gbc_pred = convert_to_one_hot(gbc_pred)
 print('gbc_pred.shape=', gbc_pred.shape )<find_best_params>
params = {
'max_depth': 7,
'gamma' :0,
'eta':.03,
'subsample': 1,
'colsample_bytree': 0.9,
'objective':'reg:linear',
'eval_metric':'rmse',
'silent': 0
}
f1s = []
alphas = np.linspace(0,1,101)
 for alpha in tqdm(alphas):
y_pred = alpha*mlp_pred +(1 - alpha)*gbc_pred
f1 = f1_score(y_test, np.argmax(y_pred, axis=-1), average='macro')
 f1s.append(f1)
 f1s = np.array(f1s)
 best_alpha = alphas[np.argmax(f1s)]
print('best_f1=', f1s.max())
 print('best_alpha=', best_alpha )<compute_test_metric>
def XGBmodel(X_train,X_test,y_train,y_test,params):
matrix_train = xgb.DMatrix(X_train,label=y_train)
 matrix_test = xgb.DMatrix(X_test,label=y_test)
 model=xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(X_train,X_test,y_train,y_test,params )
def plot_cm(y_true, y_pred, title):
figsize=(16,16)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap='viridis', annot=annot, fmt='', ax=ax )<load_from_csv>
prediction = model.predict(xgb.DMatrix(df_test), ntree_limit = model.best_ntree_limit ).tolist()
def submit_result(mlp, gbc, alpha):
print('Reading data...')
 data = pd.read_csv('.. /input/clean-2apply-kalman/test_clean_2apply_kalman.csv')
 print('Feature engineering...')
 df = divide_and_add_features(data['signal'])
 print('Predicting MLP...')
 mlp_pred = mlp.predict(df.values)
 print('Predicting GBC...')
 gbc_pred = gbc.predict(df.values)
 gbc_pred = convert_to_one_hot(gbc_pred)
 print('Ensembling...')
 y_pred = alpha*mlp_pred +(1 - alpha)*gbc_pred
y_pred = np.argmax(y_pred, axis=-1)
 print('Writing submission...')
 submission = pd.DataFrame()
submission['time'] = data['time']
submission['open_channels'] = y_pred
submission.to_csv('submission.csv', index=False, float_format='%.4f')
 print('Submission finished!' )<compute_test_metric>
test = pd.read_csv(".. /input/test.csv")
 holdout = pd.DataFrame({'key': test['key'], 'fare_amount': prediction})
 holdout.to_csv('xgb_with4mtrainedrows.csv', index=False )
submit_result(mlp, gbc, best_alpha )<set_options>
!pip install tensorflow_addons
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>