Kaggle competition: New York City Taxi Trip Duration
train['GameWeather'] = train['GameWeather'].apply(map_weather )<drop_column>
fp1 = os.path.join(".. ", "input", "train.csv")
 fp2 = os.path.join(".. ", "input", "test.csv" )
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )<sort_values>
train = pd.read_csv(fp1, index_col=0)
 train.head()
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()<define_variables>
test = pd.read_csv(fp2, index_col=0)
 test.head()
play_features = []
pos_features = ["X", "Y"]
player_features = ["S", "A", "Dis", "Orientation", "Dir", "PlayerHeight", "PlayerWeight", "Position", "PlayerAge", "IsRusher", "Team"]
game_features = ["YardLine", "Quarter", "GameClock",
"Down", "Distance", "HomeScoreBeforePlay", "VisitorScoreBeforePlay", "DefendersInTheBox", "Temperature", "Humidity", "TimeDelta"]<groupby>
train = train[(train.trip_duration < 5000)]
game_train = train[game_features + ["PlayId"]].groupby("PlayId" ).max() [game_features].values<prepare_x_and_y>
train = train.loc[(train['pickup_longitude'] > -75)&(train['pickup_longitude'] < -73)]
train = train.loc[(train['pickup_latitude'] > 40)&(train['pickup_latitude'] < 41)]
scaling_factor = 2
X_train = np.zeros(( len(train)//22, 120//scaling_factor, 58//scaling_factor, len(player_features)+2), dtype = np.float16 )<prepare_x_and_y>
train = train.loc[(train['dropoff_longitude'] > -75)&(train['dropoff_longitude'] < -73)]
train = train.loc[(train['dropoff_latitude'] > 40.5)&(train['dropoff_latitude'] < 41.5)]
x_field = np.arange(-(120//scaling_factor)/2,(120//scaling_factor)/2, 1)
 y_field = np.abs(np.arange(-(58//scaling_factor)/2,(58//scaling_factor)/2, 1))<compute_test_metric>
train = train.loc[(train['passenger_count'] >= 0)&(train['passenger_count'] <= 6)]
field_array = np.dot(x_field[:, None],y_field[None,:])
 field_array = field_array/field_array.max()<feature_engineering>
train.isnull().sum()
X_train[:, :, :, -2] = field_array<data_type_conversions>
train.isnull().sum()
plt.imshow(X_train[0, :, :, -2].astype(np.float32 ).T )<categorify>
train.duplicated().sum()
le = LabelEncoder()
train["Position"] = le.fit_transform(train["Position"])
 train[["X", "Y"]] = train[["X", "Y"]].astype(int )<normalization>
train = train.drop_duplicates()
train.duplicated().sum()
scalers = {}
for feature in player_features:
if feature == "X" or feature == "Y" or feature == "IsRusher":
continue
else:
ss = StandardScaler()
train[feature] = ss.fit_transform(train[feature].values.reshape(-1, 1))
scalers[feature] = ss<data_type_conversions>
train.drop(["store_and_fwd_flag"], axis=1, inplace=True)
 test.drop(["store_and_fwd_flag"], axis=1, inplace=True )
for i, playid in enumerate(tqdm(range(len(train)))) :
player_df = train.iloc[i, :]
X_train[i//22, player_df["X"]//scaling_factor, player_df["Y"]//scaling_factor, 0:len(player_features)] += player_df[player_features].astype(np.float16)
 X_train[i//22, player_df["YardLine"]//2, :, -1] = 1<filter>
plg, plt = 'pickup_longitude', 'pickup_latitude'
dlg, dlt = 'dropoff_longitude', 'dropoff_latitude'
pdt, ddt = 'pickup_datetime', 'dropoff_datetime'
X_train[np.isnan(X_train)] = 0
game_train[np.isnan(game_train)] = 0<prepare_x_and_y>
def haversine(lon1, lat1, lon2, lat2):

 lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat/2)**2 + cos(lat1)* cos(lat2)* sin(dlon/2)**2
c = 2 * asin(sqrt(a))
km = 6371* c
return km
def euclidian_distance(x):
x1, y1 = np.float64(x[plg]), np.float64(x[plt])
 x2, y2 = np.float64(x[dlg]), np.float64(x[dlt])
 return haversine(x1, y1, x2, y2 )
y_train = np.zeros(shape=(train.shape[0]//22, 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))<data_type_conversions>
%time
train['distance'] = train[[plg, plt, dlg, dlt]].apply(euclidian_distance, axis=1 )
for i in range(len(player_features)+2):
plt.imshow(X_train[0, :, :, i].astype(np.float32 ).T)
 try:
print(player_features[i])
 except:
pass
plt.show()<choose_model_class>
%time
test['distance'] = test[[plg, plt, dlg, dlt]].apply(euclidian_distance, axis=1 )
__all__ = ['RAdam']
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
train[pdt] = train[pdt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
train[ddt] = train[ddt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
class Mish(Activation):

 def __init__(self, activation, **kwargs):
super(Mish, self ).__init__(activation, **kwargs)
 self.__name__ = 'Mish'
def mish(inputs):
return inputs * tf.math.tanh(tf.math.softplus(inputs))
get_custom_objects().update({'Mish': Mish(mish)} )<compute_test_metric>
test[pdt] = test[pdt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )<choose_model_class>
train['month'] = train[pdt].apply(lambda x : x.month)
 train['week_day'] = train[pdt].apply(lambda x : x.weekday())
 train['day_month'] = train[pdt].apply(lambda x : x.day)
 train['pickup_time_minutes'] = train[pdt].apply(lambda x : x.hour * 60.0 + x.minute )
def create_model() :
image_inputs = Input(shape=(X_train.shape[1],X_train.shape[2], X_train.shape[3]))
game_inputs = Input(shape=(game_train.shape[1],))
x = BatchNormalization()(image_inputs)
 x = Conv2D(16,(3,3), activation='Mish' )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Conv2D(16,(3,3), activation='Mish' )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Conv2D(32,(3,3), activation='Mish' )(x)
 x = Conv2D(64,(3,3), activation='Mish' )(x)
 x = GlobalMaxPooling2D()(x)
 game_feats = BatchNormalization()(game_inputs)
 game_feats = Dense(25, activation = 'Mish' )(game_feats)
 x = Concatenate()([x, game_feats])
 x = Dropout (.2 )(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inputs = [image_inputs,game_inputs], outputs=out)
 return model<train_model>
test['month'] = test[pdt].apply(lambda x : x.month)
 test['week_day'] = test[pdt].apply(lambda x : x.weekday())
 test['day_month'] = test[pdt].apply(lambda x : x.day)
 test['pickup_time_minutes'] = test[pdt].apply(lambda x : x.hour * 60.0 + x.minute )
def train_model(x_tr, y_tr, x_vl, y_vl):
model = create_model()
print(model.summary())
 er = EarlyStopping(patience=5, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss=crps, metrics=[metrics.mae])
 model.fit(x_tr, y_tr, epochs=100, callbacks=[er], validation_data=[x_vl, y_vl])
 return model<data_type_conversions>
features_train = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "distance", "month", "week_day", "day_month", "pickup_time_minutes"]
X_train = train[features_train]
y_train = np.log1p(train["trip_duration"])
 features_test = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "distance", "month", "week_day", "day_month", "pickup_time_minutes"]
X_test = test[features_test]
rkf = RepeatedKFold(n_splits=4, n_repeats=1)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx].astype(np.float16), y_train[tr_idx].astype(np.float16)
 x_vl, y_vl = X_train[vl_idx].astype(np.float16), y_train[vl_idx].astype(np.float16)
 game_tr, game_vl = game_train[tr_idx], game_train[vl_idx]
model = train_model([x_tr,game_tr], y_tr, [x_vl,game_vl], y_vl)
 models.append(model )<predict_on_test>
from sklearn.ensemble import RandomForestRegressor
preds = models[0].predict([x_vl,game_vl] )<drop_column>
del train, X_train<feature_engineering>
rf = RandomForestRegressor(n_estimators=100, min_samples_leaf=2, max_features='auto', max_depth=50, bootstrap=True, n_jobs=-1 )
def make_pred(df, sample, env, models):
train = df
train = fix_play_direction(train)
 train['GameClock'] = train['GameClock'].apply(strtoseconds)
 train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(str_to_float)
 train.drop('WindDirection', axis=1, inplace=True)
 train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 train['Team'] = train['Team'].apply(lambda x: x.strip() =='home')
 train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(map_weather)
 train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True)
 train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
X_train = np.zeros(( 1, 120//scaling_factor, 58//scaling_factor, len(player_features)+2), dtype = np.float16)
 X_train[:, :, :, -2] = field_array
train["Position"] = le.transform(train["Position"])
 train[["X", "Y"]] = train[["X", "Y"]].astype(int)
 for feature in player_features:
if feature == "X" or feature == "Y" or feature == "IsRusher":
continue
else:
ss = scalers[feature]
train[feature] = ss.transform(train[feature].values.reshape(-1, 1))
for row in range(22):
player_df = train.iloc[row]
X_train[0, player_df["X"]//scaling_factor, player_df["Y"]//scaling_factor, 0:len(player_features)] = player_df[player_features].astype(np.float16)
 X_train[0, train["YardLine"]//2, :, -1] = 1
game_train = train[game_features + ["PlayId"]].groupby("PlayId" ).max() [game_features].values
game_train[np.isnan(game_train)] = 0
X_train[np.isnan(X_train)] = 0
y_pred = np.mean([model.predict([X_train, game_train])for model in models], axis=0)
 y_pred[0] = np.cumsum(y_pred[0], axis=0 ).clip(0, 1)
 y_pred[:, -1] = np.ones(shape=(y_pred.shape[0], 1))
y_pred[:, 0] = np.zeros(shape=(y_pred.shape[0], 1))
env.predict(pd.DataFrame(data=y_pred,columns=sample.columns))
return y_pred<find_best_params>
rf.fit(X_train, y_train )
env = nflrush.make_env()<predict_on_test>
log_pred = rf.predict(X_test)
 y_pred = np.exp(log_pred)- np.ones(len(log_pred))
for test, sample in tqdm(env.iter_test()):
make_pred(test, sample, env, models )<save_to_csv>
my_submission = pd.DataFrame({'id': test.index, 'trip_duration': y_pred})
 my_submission.head()
<define_search_space><EOS> 
my_submission.to_csv("submission.csv", index=False )