Kaggle competition: New York City Taxi Fare Prediction
print('Wavenet score {0:.4f}'.format(np.mean(f1_score(y, np.round(np.clip(y_pred_train_wnet,0,10)).astype(int), average="macro"))))<compute_test_metric>
n_train = 1_000_000
n_features = 2
dtype_map = {
'fare_amount': np.float32,
'pickup_longitude': np.float64,
'pickup_latitude': np.float64,
'dropoff_longitude': np.float64,
'dropoff_latitude': np.float64,
'passenger_count': np.int8
}
df = pd.read_csv(".. /input/train.csv", nrows = n_train, parse_dates=["pickup_datetime"], dtype=dtype_map)
 df_test = pd.read_csv('.. /input/test.csv', parse_dates=["pickup_datetime"], dtype=dtype_map )
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )<set_options>
def add_travel_vector_features(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
df['circ_diff'] = np.sqrt(df.abs_diff_longitude**2 + df.abs_diff_latitude**2)
 add_travel_vector_features(df)
 add_travel_vector_features(df_test)
 print(df.isnull().sum())
 print('Old size: %d' % len(df))
df = df.dropna(how='any', axis='rows')
 print('New size: %d' % len(df))
gc.collect()<drop_column>
min_year = df.pickup_datetime.apply(lambda t: t.year ).min()
df['pickup_year'] = df.pickup_datetime.apply(lambda t: t.year - min_year)
 df['pickup_hour'] = df.pickup_datetime.apply(lambda t: t.hour)
 df['pickup_day'] = df.pickup_datetime.apply(lambda t: t.timetuple().tm_yday)
 df_test['pickup_year'] = df_test.pickup_datetime.apply(lambda t: t.year - min_year)
 df_test['pickup_hour'] = df_test.pickup_datetime.apply(lambda t: t.hour)
 df_test['pickup_day'] = df_test.pickup_datetime.apply(lambda t: t.timetuple().tm_yday)
del y_pred_train_lgb, y_pred_train_wnet
gc.collect()<define_variables>
df_train, df_val = train_test_split(df, test_size=0.1)
 len(df_val )
y_preds = w_lgb*y_lgb_pred + w_wnet*y_wnet_pred<drop_column>
def get_input_matrix(df):
return np.column_stack(( df.abs_diff_longitude, df.abs_diff_latitude, df.circ_diff,
df.dropoff_longitude, df.dropoff_latitude,
df.pickup_longitude, df.pickup_latitude,
df.passenger_count, df.pickup_year,
df.pickup_hour, df.pickup_day))
x_train, x_val = get_input_matrix(df_train), get_input_matrix(df_val)
 y_train, y_val = np.array(df_train.fare_amount), np.array(df_val.fare_amount )
del y_lgb_pred, y_wnet_pred
gc.collect()<categorify>
reg = RandomForestRegressor(max_depth=28, n_estimators=500, oob_score=True, n_jobs=-1,min_samples_split=10, verbose=1 )
def pred_proc(pred):
pred = np.round(np.clip(pred, 0, 10))
return pred.astype(int )<predict_on_test>
reg.fit(x_train, y_train )
y_preds = pred_proc(y_preds)
 y_train_preds = pred_proc(y_train_preds )<compute_test_metric>
reg.oob_score_
print('The main solution score {0:.4f}'.format(np.mean(f1_score(y, y_train_preds, average="macro"))))<set_options>
y_pred = reg.predict(x_val)
del y_train_preds
gc.collect()<save_to_csv>
score = r2_score(y_val, y_pred)
 score
test['open_channels'] = y_preds
test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )<define_variables>
reg.score(x_val, y_val )
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'<import_modules>
mean_squared_error(y_val, y_pred )
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error<set_options>
x_test = get_input_matrix(df_test )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<feature_engineering>
predictions = reg.predict(x_test )
<load_from_disk><EOS> 
RFSubmission = pd.DataFrame({ 'key': df_test.key.ravel() ,
'fare_amount': predictions })
 RFSubmission.to_csv("RFSubmission.csv", index=False )