Kaggle competition: New York City Taxi Fare Prediction
( train_label, train_features)=(train["label"], train.drop(["label"],axis=1)) 
(test_id, test_features)=(test["id"], test.drop(["id"],axis=1)) 
(train_x, train_y)=(train_features.values, train_label.values)
 test_x = test_features.values<split>
plt.style.use('dark_background')
 sns.set_style("darkgrid")
train_x, valid_x, train_y, valid_y = train_test_split(train_x, train_y, test_size = 0.05, random_state=42 )<categorify>
%%time
train_path = '.. /input/train.csv'
traintypes = {'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
cols = list(traintypes.keys())
 train_df = pd.read_csv(train_path, usecols=cols, dtype=traintypes, nrows=2_000_000 )
normalized_train_x = reshaped_train_x/255
normalized_valid_x = reshaped_valid_x/255
normalized_test_x = reshaped_test_x/255
train_labels = keras.utils.to_categorical(train_y, 10)
 valid_labels = keras.utils.to_categorical(valid_y, 10 )<create_dataframe>
%%time
train_df.to_feather('nyc_taxi_data_raw.feather' )
datagen_train = ImageDataGenerator(rotation_range=15,
width_shift_range = 0.2,
height_shift_range = 0.2,
shear_range = 25,
zoom_range = 0.4,)
 datagen_valid = ImageDataGenerator()<choose_model_class>
df_train = pd.read_feather('nyc_taxi_data_raw.feather')
model = Sequential()
model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256))
model.add(BatchNormalization())
 model.add(Dense(128))
model.add(BatchNormalization())
 model.add(Dense(10, activation='softmax'))
model.summary()<choose_model_class>
len(df_train[df_train.fare_amount > 0] )
lr = 2e-3
batch_size=128
epochs=50
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=5,
verbose=1,
factor=0.2)
 earlystop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=8 )<choose_model_class>
df_train = df_train[df_train.fare_amount>=0]
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
df_train.isnull().sum()
history = model.fit(datagen_train.flow(normalized_train_x, train_labels, batch_size=batch_size),
epochs = epochs,
steps_per_epoch=normalized_train_x.shape[0] // batch_size,
validation_data =(normalized_valid_x,valid_labels),
shuffle=True,
callbacks=[learning_rate_reduction, earlystop] )<predict_on_test>
df_train = df_train.dropna(how = 'any', axis = 'rows')
prediction = model.predict(normalized_test_x )<save_to_csv>
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head(5 )
test['label'] = np.argmax(prediction, axis=1)
 submission = test[['id', 'label']]
submission.to_csv("submission.csv",index=False )<set_options>
df_train['pickup_datetime'] = pd.to_datetime(df_train['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC" )
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50)
<compute_test_metric>
df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC" )
def macro_f1_score(y_true, y_pred, n_labels):
total_f1 = 0.
 for i in range(n_labels):
yt = y_true == i
yp = y_pred == i
tp = np.sum(yt & yp)
 tpfp = np.sum(yp)
 tpfn = np.sum(yt)
 if tpfp == 0:
print('[WARNING] F-score is ill-defined and being set to 0.0 in labels with no predicted samples.')
 precision = 0.
 else:
precision = tp / tpfp
if tpfn == 0:
print(f'[ERROR] label not found in y_true...')
 recall = 0.
 else:
recall = tp / tpfn
if precision == 0.or recall == 0.:
f1 = 0.
 else:
f1 = 2 * precision * recall /(precision + recall)
 total_f1 += f1
return total_f1 / n_labels
macro_f1_score_nb = nb.jit(nb.float64(nb.int32[:], nb.int32[:], nb.int64), nopython=True, nogil=True )(macro_f1_score )<load_from_disk>
def add_new_date_time_features(dataset):
dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['year'] = dataset.pickup_datetime.dt.year
dataset['day_of_week'] = dataset.pickup_datetime.dt.dayofweek
return dataset
%%time
datadir = Path('/kaggle/input/liverpool-ion-switching-feather')
 train = pd.read_feather(datadir/'train.feather')
 test = pd.read_feather(datadir/'test.feather')
 submission = pd.read_feather(datadir/'sample_submission.feather' )<define_variables>
df_train = add_new_date_time_features(df_train)
 df_test = add_new_date_time_features(df_test )
signal_array = train['signal'].values
open_channels = train['open_channels'].values
test_signal_array = test['signal'].values<define_variables>
def calculate_abs_different(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
calculate_abs_different(df_train)
 calculate_abs_different(df_test )
debug = False
arch = 'lgb'
outdir = Path('.' )<feature_engineering>
def convert_different_miles(df):
df['abs_diff_longitude'] = df.abs_diff_longitude*50
df['abs_diff_latitude'] = df.abs_diff_latitude*69
convert_different_miles(df_train)
 convert_different_miles(df_test )
def clean_train(train):
def f(x, low, high, mid):
return -(( -low + high)/ 625)*(x - mid)** 2 + high - low
a = 500000
b = 600000
train.loc[train.index[a:b], 'signal'] = train.signal[a:b].values - 3 *(train.time.values[a:b] - 50)/ 10.
 batch = 7
a = 500000 *(batch - 1)
 b = 500000 * batch
train.loc[train.index[a:b], 'signal'] = train.signal.values[a:b] - f(train.time[a:b].values, -1.817, 3.186, 325)
 batch = 8
a = 500000 *(batch - 1)
 b = 500000 * batch
train.loc[train.index[a:b], 'signal'] = train.signal.values[a:b] - f(train.time[a:b].values, -0.094, 4.936, 375)
 batch = 9
a = 500000 *(batch - 1)
 b = 500000 * batch
train.loc[train.index[a:b], 'signal'] = train.signal.values[a:b] - f(train.time[a:b].values, 1.715, 6.689, 425)
 batch = 10
a = 500000 *(batch - 1)
 b = 500000 * batch
train.loc[train.index[a:b], 'signal'] = train.signal.values[a:b] - f(train.time[a:b].values, 3.361, 8.45, 475)
 return train
def clean_test(test):
start = 500
a = 0
b = 100000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 start = 510
a = 100000
b = 200000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 start = 540
a = 400000
b = 500000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 start = 560
a = 600000
b = 700000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 start = 570
a = 700000
b = 800000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 start = 580
a = 800000
b = 900000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - 3 *(test.time.values[a:b] - start)/ 10.
 def f(x):
return -(0.00788)*(x - 625)** 2 + 2.345 + 2.58
a = 1000000
b = 1500000
test.loc[test.index[a:b], 'signal'] = test.signal.values[a:b] - f(test.time[a:b].values)
 return test
def plot_all_train_signal(train, title):
plt.figure(figsize=(20,5)) ; res = 1000
plt.plot(range(0,train.shape[0],res),train.signal[0::res])
 for i in range(11): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for j in range(10): plt.text(j*500000+200000,10,str(j+1),size=20)
 plt.xlabel('Row',size=16); plt.ylabel('Signal',size=16);
plt.title(title,size=20)
 plt.show()
def plot_all_test_signal(test, title):
let = ['A','B','C','D','E','F','G','H','I','J']
plt.figure(figsize=(20,5))
res = 1000
plt.plot(range(0,test.shape[0],res),test.signal[0::res])
 for i in range(5): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for i in range(21): plt.plot([i*100000,i*100000],[-5,12.5],'r:')
 for k in range(4): plt.text(k*500000+250000,10,str(k+1),size=20)
 for k in range(10): plt.text(k*100000+40000,7.5,let[k],size=16)
 plt.title(title,size=20)
 plt.show()<split>
meas_ang = 0.506
def add_distance(df):
df['Euclidean'] =(df.abs_diff_latitude**2 + df.abs_diff_longitude**2)**0.5
df['delta_manh_long'] =(df.Euclidean*np.sin(np.arctan(df.abs_diff_longitude / df.abs_diff_latitude)-meas_ang)).abs()
df['delta_manh_lat'] =(df.Euclidean*np.cos(np.arctan(df.abs_diff_longitude / df.abs_diff_latitude)-meas_ang)).abs()
df['distance'] = df.delta_manh_long + df.delta_manh_lat
df.drop(['abs_diff_longitude', 'abs_diff_latitude','Euclidean', 'delta_manh_long', 'delta_manh_lat'], axis=1, inplace=True)
 add_distance(df_train)
 add_distance(df_test )
if debug:
s = 500000*4
train = train.iloc[s:s+50000]
test = test.iloc[:1000]
submission = submission.iloc[:1000]<categorify>
def calculate_direction(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):

 R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = pickup_lon - dropoff_lon
a = np.arctan2(np.sin(dlon * np.cos(dropoff_lat)) ,np.cos(pickup_lat)* np.sin(dropoff_lat)- np.sin(pickup_lat)* np.cos(dropoff_lat)* np.cos(dlon))
return a
def filter_wave(x, cutoff=(-1, -1), N=4, filtering='lfilter'):

 assert x.ndim == 1
output = 'sos' if filtering == 'sos' else 'ba'
if cutoff[0] <= 0 and cutoff[1] <= 0:
return x
elif cutoff[0] <= 0 and cutoff[1] > 0:
output = signal.butter(N, Wn=cutoff[1]/len(x), btype='lowpass', output=output)
 elif cutoff[0] > 0 and cutoff[1] <= 0:
output = signal.butter(N, Wn=cutoff[0]/len(x), btype='highpass', output=output)
 else:
output = signal.butter(N, Wn=(cutoff[0]/len(x), cutoff[1]/len(x)) , btype='bandpass', output=output)
 if filtering == 'lfilter':
b, a = output
return signal.lfilter(b, a, x)
 elif filtering == 'filtfilt':
b, a = output
return signal.filtfilt(b, a, x)
 elif filtering == 'sos':
sos = output
return signal.sosfilt(sos, x)
 else:
raise ValueError("[ERROR] Unexpected value filtering={}".format(filtering))<define_variables>
train_df['direction'] = calculate_direction(train_df['pickup_latitude'].values, train_df['pickup_longitude'].values,
train_df['dropoff_latitude'].values , train_df['dropoff_longitude'].values)
 df_test['direction'] = calculate_direction(df_test['pickup_latitude'].values, df_test['pickup_longitude'].values,
df_test['dropoff_latitude'].values , df_test['dropoff_longitude'].values )
train_separate_indices = [
0, 500_000, 600_000, 1_000000, 1_500000, 2_000000, 2_500000,
3_000000, 3_500000, 4_000000, 4_500000, 5_000000]
test_separate_indices = [
0, 100_000, 200_000, 300_000, 400_000, 500_000, 600_000, 700_000,
800_000, 900_000, 1_000000, 1_500000, 2_000000]
def _create_total_batch_indices(separate_indices):
s = separate_indices
total_batch = np.zeros(( s[-1],), dtype=np.int64)
 for i in range(len(s)- 1):
total_batch[s[i]:s[i+1]] = i
return total_batch
def create_total_batch_indices(train: bool = True):
separate_indices = train_separate_indices if train else test_separate_indices
return _create_total_batch_indices(separate_indices)
 class Preprocessor(object):

 def __init__(self, feature_cols=None, debug=False, use_sep=True):
self.feature_cols = feature_cols
self.debug = debug
self.use_sep = use_sep
self.window_list = [5000, 10000, 50000]
def preprocess(self, tt, train=True):
feature_cols = self.feature_cols
window_list = self.window_list
window_str = '-'.join(list(map(str, window_list)))
 cache_path = f'pp_tt_train{int(train)}_debug{int(self.debug)}_usesep{int(self.use_sep)}_w{window_str}.feather'
msignal_cols = []
for window in window_list:
msignal_cols.extend([
f'signal_batch_{window}_min', f'signal_batch_{window}_max',
f'signal_batch_{window}_std', f'signal_batch_{window}_mean',
f'mean_abs_chg_batch_{window}',
f'abs_max_batch_{window}', f'abs_min_batch_{window}',
f'range_batch_{window}', f'maxtomin_batch_{window}', f'abs_avg_batch_{window}'
])
 msignal_cols.extend([
'signal_shift+1', 'signal_shift-1',
'signal_shift+2', 'signal_shift-2'])
 default_feature_cols = ['signal', 'baseline']
default_feature_cols.extend(msignal_cols)
 default_feature_cols.extend([f'{c}_msignal' for c in msignal_cols])
 default_feature_cols.extend([f'{c}_mbase' for c in msignal_cols])
 if not os.path.exists(cache_path):
tt = tt.sort_values(by=['time'] ).reset_index(drop=True)
 tt.index =(( tt.time * 10_000)- 1 ).values
if self.use_sep:
separate_indices = train_separate_indices if train else test_separate_indices
separate_indices = separate_indices[:len(tt)]
tt['total_batch'] = create_total_batch_indices(train=train)[:len(tt)]
else:
raise NotImplementedError()
separate_indices = None
tt['total_batch'] = tt.index // 500_000
tt['batch'] = tt.index // 50_000
tt['batch_index'] = tt.index -(tt.batch * 50_000)
 tt['batch_slices'] = tt['batch_index'] // 5_000
tt['batch_slices2'] = tt.apply(lambda r: '_'.join(
 [str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 cutoff_high = 100
signal = tt['signal'].values
total_batch = tt['total_batch'].values
smooth_fn = lambda x: filter_wave(x, cutoff=(0, cutoff_high), filtering='filtfilt')
 baseline = np.concatenate([smooth_fn(signal[separate_indices[k]:separate_indices[k+1]])
 for k in range(total_batch[-1] + 1)])
 tt['baseline'] = baseline
for window in window_list:
rolling_df = tt.groupby('total_batch')['signal'].rolling(
 window=window, min_periods=1, center=True)
 tt[f'signal_batch_{window}_min'] = rolling_df.min().values
tt[f'signal_batch_{window}_max'] = rolling_df.max().values
tt[f'signal_batch_{window}_std'] = rolling_df.std().values
tt[f'signal_batch_{window}_mean'] = rolling_df.mean().values
tt[f'mean_abs_chg_batch_{window}'] = rolling_df.apply(
 lambda x: np.mean(np.abs(np.diff(x))), raw=True ).values
tt[f'abs_max_batch_{window}'] = rolling_df.apply(
 lambda x: np.max(np.abs(x)) , raw=True ).values
tt[f'abs_min_batch_{window}'] = rolling_df.apply(
 lambda x: np.min(np.abs(x)) , raw=True ).values
tt[f'range_batch_{window}'] = \
tt[f'signal_batch_{window}_max'] - tt[f'signal_batch_{window}_min']
tt[f'maxtomin_batch_{window}'] = \
tt[f'signal_batch_{window}_max'] / tt[f'signal_batch_{window}_min']
tt[f'abs_avg_batch_{window}'] = \ 
(tt[f'abs_min_batch_{window}'] + tt[f'abs_max_batch_{window}'])/ 2
tt['signal_shift+1'] = tt.groupby(['batch'] ).shift(1, fill_value=0.) ['signal']
tt['signal_shift-1'] = tt.groupby(['batch'] ).shift(-1, fill_value=0.) ['signal']
tt['signal_shift+2'] = tt.groupby(['batch'] ).shift(2, fill_value=0.) ['signal']
tt['signal_shift-2'] = tt.groupby(['batch'] ).shift(-2, fill_value=0.) ['signal']
for c in msignal_cols:
tt[f'{c}_msignal'] = tt[c] - tt['signal']
for c in msignal_cols:
tt[f'{c}_mbase'] = tt[c] - tt['baseline']
tt = tt.replace([np.inf, -np.inf], np.nan)
 print('Number of nan', tt.isna().sum().sum())
 tt.fillna(-1, inplace=True)
 print(f'saving to {cache_path}')
 else:
print(f'loading from {cache_path}')
 tt = pd.read_feather(cache_path)
 if feature_cols is None:
feature_cols = default_feature_cols
X_train = tt[feature_cols]
if train:
y_train = tt['open_channels'].values
else:
y_train = None
return X_train, y_train 
<prepare_x_and_y>
train_df['pickup_latitude'].apply(lambda x: np.radians(x))
train_df['pickup_longitude'].apply(lambda x: np.radians(x))
train_df['dropoff_latitude'].apply(lambda x: np.radians(x))
train_df['dropoff_longitude'].apply(lambda x: np.radians(x))
df_test['pickup_latitude'].apply(lambda x: np.radians(x))
df_test['pickup_longitude'].apply(lambda x: np.radians(x))
df_test['dropoff_latitude'].apply(lambda x: np.radians(x))
df_test['dropoff_longitude'].apply(lambda x: np.radians(x))
pp = Preprocessor(debug=debug)
 with timer('preprocess train'):
X_train, y_train = pp.preprocess(train )<predict_on_test>
df_train.drop(columns=['pickup_datetime'], inplace=True)
 y = df_train['fare_amount']
df_train = df_train.drop(columns=['fare_amount'] )
class F1MacroScorerWrapper(object):

 def fit(self, X, y):
print('F1MacroScorerWrapper fit called!')
 def predict(self, X):
raise NotImplementedError()
def score(self, X, y):
y_pred = self.predict(X)
 y_pred_valid_int = np.round(np.clip(y_pred, 0, 10))
macro_f1 = macro_f1_score_nb(y.astype(np.int32), y_pred_valid_int.astype(np.int32), 10)
 return macro_f1
class LGBWrapper(F1MacroScorerWrapper):
def __init__(self, lgb_model):
self.lgb_model = lgb_model
def predict(self, X):
return self.lgb_model.predict(X, num_iteration=self.lgb_model.best_iteration)
 def feature_importance(self, **kwargs):
return self.lgb_model.feature_importance(**kwargs)
 def calc_permutation_importance(model, valid_data, feature_names=None):

 perm = PermutationImportance(
 model, random_state=1, cv='prefit' ).fit(*valid_data)
 perm_importance_df = pd.DataFrame({
'importance': perm.feature_importances_
}, index=feature_names ).sort_values('importance', ascending=True)
 return perm, perm_importance_df<prepare_x_and_y>
x_train,x_test,y_train,y_test = train_test_split(df_train,y,random_state=123,test_size=0.1 )
def fit_lgb(train, val, devices=(-1,), seed=None, cat_features=None,
num_rounds=1500, lr=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l1'
params = {'num_leaves': 128,
'min_data_in_leaf': 64,
'objective': 'huber',
'max_depth': -1,
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"metric": metric,
"verbosity": -1,
'reg_alpha': 0.1,
'reg_lambda': 0.3
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training'][metric],
'valid/mae': model.best_score['valid_1'][metric]}
return model, y_pred_valid, log<train_model>
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'nthread': 4,
'num_leaves': 31,
'learning_rate': 0.1,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 10000 ,
'bagging_freq': 10,
'metric': 'rmse',
'zero_as_missing': True,
'num_rounds':50000
}
calc_perm = True
y_valid_pred_total = np.zeros(X_train.shape[0])
 models = []
log_list = []
macro_f1_list = []
gain_importance_list = []
split_importance_list = []
perm_importance_list = []
for fold_idx,(train_idx, valid_idx)in enumerate(cv_iter):
train_data = X_train.iloc[train_idx, :], y_train[train_idx]
valid_data = X_train.iloc[valid_idx, :], y_train[valid_idx]
print('fold_idx', fold_idx, 'train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgb(train_data, valid_data,
num_rounds=5000, lr=0.3)
 model = LGBWrapper(model)
 print('log', log)
 y_valid_pred_total[valid_idx] = y_pred_valid
models.append(model)
 y_true_valid = valid_data[1]
y_pred_valid_int = np.round(np.clip(y_pred_valid, 0, 10))
macro_f1 = macro_f1_score(y_true_valid, y_pred_valid_int, 10)
 print('macro_f1', macro_f1)
 log['fold_idx'] = fold_idx
log['macro_f1'] = macro_f1
log_list.append(log)
 macro_f1_list.append(macro_f1)
 feature_names = train_data[0].columns.tolist()
gain_importance_df = calc_model_importance(
 model, feature_names=feature_names, importance_type='gain')
 gain_importance_list.append(gain_importance_df)
 split_importance_df = calc_model_importance(
 model, feature_names=feature_names, importance_type='split')
 split_importance_list.append(split_importance_df)
 if calc_perm:
print('calculating permutation importance...')
 if(debug and fold_idx == 0)or not debug:
with timer('calc_permutation_importance'):
perm, perm_importance_df = calc_permutation_importance(
 model, valid_data, feature_names=feature_names)
 perm_importance_list.append(perm_importance_df)
 gc.collect()<save_to_csv>
train_set = lgbm.Dataset(x_train, y_train, silent=False,categorical_feature=['year','month','day','day_of_week'])
 valid_set = lgbm.Dataset(x_test, y_test, silent=False,categorical_feature=['year','month','day','day_of_week'])
 model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=1000,verbose_eval=500, valid_sets=valid_set )
log_df = pd.DataFrame(log_list)
 log_df.to_csv(outdir/'log.csv', index=False)
 logmean_df = log_df.mean(axis=0)
 logmean_df.to_csv(outdir/'log_mean.csv' )<compute_test_metric>
test_key = df_test['key']
df_test.drop(columns=["pickup_datetime",'key'], axis=1, inplace=True )
def calc_mean_importance(importance_df_list):
mean_importance = np.mean(
 np.array([df['importance'].values for df in importance_df_list]), axis=0)
 mean_df = importance_df_list[0].copy()
mean_df['importance'] = mean_importance
return mean_df<save_to_csv>
prediction = model.predict(df_test, num_iteration = model.best_iteration)
mean_gain_df = calc_mean_importance(gain_importance_list)
 plot_importance(mean_gain_df, title='Model feature importance by gain')
 mean_gain_df = mean_gain_df.reset_index().rename(columns={'index': 'feature_names'})
 mean_gain_df.to_csv(outdir / 'gain_importance_mean.csv', index=False )<save_to_csv>
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction
})
 submission.to_csv('taxi_fare_submission.csv',index=False )