Kaggle competition: New York City Taxi Trip Duration
COMPONENTS = 20
methods = [
TruncatedSVD(n_components=COMPONENTS),
PCA(n_components=COMPONENTS),
GaussianRandomProjection(n_components=COMPONENTS, eps=0.1, random_state = SEED + 354),
]
embeddings = []
for method in methods:
name = method.__class__.__name__
embeddings.append(
 pd.DataFrame(method.fit_transform(df), columns=[f"{name}_{i}" for i in range(COMPONENTS)])
)
 print(f">> Ran {name}")
 X = pd.concat(embeddings, axis=1 ).reset_index(drop=True)
 del embeddings, method, methods, name
gc.collect() ;<feature_engineering>
df = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
%%time
dropzero = []
for row in np.arange(df.shape[0]):
dropzero.append(df.values[row, np.nonzero(df.values[row])])
 additional_df = pd.DataFrame(index=df.index)
 additional_df['mean'] = [x.mean() for x in dropzero]
additional_df['std'] = [x.std(ddof=1)for x in dropzero]
additional_df['var'] = [x.var(ddof=1)for x in dropzero]
additional_df['q_0_25'] = [np.percentile(x, q = 25)for x in dropzero]
additional_df['q_0_50'] = [np.percentile(x, q = 50)for x in dropzero]
additional_df['q_0_75'] = [np.percentile(x, q = 75)for x in dropzero]<correct_missing_values>
one = df.groupby('vendor_id' ).count() ['id'].iloc[0:1]
two = df.groupby('vendor_id' ).count() ['id'].iloc[1:2]
print('Vendor_id 2 has received ' , np.subtract(two, one ).values[0], 'more bookings then Vendor_id 1.' )
additional_df.fillna(additional_df.mean(axis = 1), inplace=True )<merge>
df.isna().sum()
X = X.join(additional_df ).fillna(0)
 del additional_df
gc.collect() ;<load_pretrained>
R = 6371.0
lat1 = np.radians(df['pickup_latitude'])
 lon1 = np.radians(df['pickup_longitude'])
 lat2 = np.radians(df['dropoff_latitude'])
 lon2 = np.radians(df['dropoff_longitude'])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat / 2)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
distance = R * c
distance_df = pd.DataFrame({'travel_distance_km':distance})
 df['travel_distance_km'] = distance_df
print(np.mean(distance_df))
lat1 = np.radians(test['pickup_latitude'])
 lon1 = np.radians(test['pickup_longitude'])
 lat2 = np.radians(test['dropoff_latitude'])
 lon2 = np.radians(test['dropoff_longitude'])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat / 2)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
distance = R * c
distance_df = pd.DataFrame({'travel_distance_km':distance})
 test['travel_distance_km'] = distance_df
USE_GPU = True
if USE_GPU and torch.cuda.is_available() :
print('using device: cuda')
 else:
print('using device: cpu' )<choose_model_class>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df['hour_pickup'] = np.array(df['pickup_datetime'].dt.hour)
 df['day_pickup'] = np.array(df['pickup_datetime'].dt.day_name())
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 test['hour_pickup'] = np.array(test['pickup_datetime'].dt.hour)
 test['day_pickup'] = np.array(test['pickup_datetime'].dt.day_name() )
torchm = nn.Sequential(
 nn.Linear(X.shape[1], 1024),
nn.Linear(1024, 512),
nn.Linear(512, 1))
optimizer = optim.Adam(torchm.parameters() , lr=0.001)
 criterion = nn.MSELoss()<data_type_conversions>
category_dict = {'0-5': 0,'5-10': 0, '10-15': 0, '15-20': 0, '20-25':0 }
for x in np.array(df[(df['travel_distance_km'] >= 0)&(df['travel_distance_km'] < 25)]['travel_distance_km']):
if x <= 5:
category_dict['0-5'] = category_dict['0-5'] + 1
elif x > 5 and x <= 10 :
category_dict['5-10'] = category_dict['5-10'] + 1
elif x > 10 and x <= 15 :
category_dict['10-15'] = category_dict['10-15'] + 1
elif x > 15 and x <= 20 :
category_dict['15-20'] = category_dict['15-20'] + 1
elif x > 20 and x <= 25 :
category_dict['20-25'] = category_dict['20-25'] + 1
pd.DataFrame(category_dict, index=[0] ).T.plot(kind='pie', subplots=True, figsize=(8, 8))
plt.title('Percentage of Trips Between 0 to 25 km')
 print('% of trips between 0 to 5 km for the trip distance between 0 to 25 km dataset : ', category_dict['0-5']/sum(category_dict.values()))
if USE_GPU and torch.cuda.is_available() :
X = torch.from_numpy(X.values ).cuda()
y = torch.from_numpy(y ).view(-1,1 ).cuda()
dtype = torch.cuda.FloatTensor
torchm.cuda()
else:
X = torch.from_numpy(X.values)
 y = torch.from_numpy(y ).view(-1,1)
 dtype = torch.FloatTensor
X = Variable(X ).type(dtype)
 y = Variable(y ).type(dtype )<train_on_grid>
sns.scatterplot(
 x=df[(df['travel_distance_km'] > 0)&(df['travel_distance_km'] < 25)&(df['trip_duration'] < 20000)]['trip_duration'],
y=df[(df['travel_distance_km'] > 0)&(df['travel_distance_km'] < 25)&(df['trip_duration'] < 20000)]['travel_distance_km'] )
torch.cuda.manual_seed_all(SEED)
 epoches = 126
for epoch in np.arange(epoches):
def closure() :
optimizer.zero_grad()
out = torchm(X[:y.shape[0]])
 loss = criterion(out, y)**.5
loss.backward()
return loss
loss = optimizer.step(closure)
 if epoch % 25 == 0:
print(f"RMSE for {epoch} epoch : {loss.data[0]}" )<predict_on_test>
df['day_pickup'] = df['day_pickup'].astype('category' ).cat.codes
df['store_and_fwd_flag'] = df['store_and_fwd_flag'].astype('category' ).cat.codes
test['day_pickup'] = test['day_pickup'].astype('category' ).cat.codes
test['store_and_fwd_flag'] = test['store_and_fwd_flag'].astype('category' ).cat.codes
preds = torchm(X[y.shape[0]:] )<data_type_conversions>
scaler = StandardScaler()
scaler.fit(df[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup']])
 scaled_features = scaler.transform(df[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup']])
 df_feat = pd.DataFrame(scaled_features, columns=['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup'])
 scaler_test = StandardScaler()
scaler_test.fit(test[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup']])
 scaled_features_test = scaler.transform(test[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup']])
 df_feat_test = pd.DataFrame(scaled_features_test, columns=['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude', 'store_and_fwd_flag',
'travel_distance_km', 'hour_pickup',
'day_pickup'])
 test = pd.concat([test['id'],df_feat_test], axis=1)
if USE_GPU and torch.cuda.is_available() :
preds = preds.cpu().data.numpy()
else:
preds = preds.data.numpy()<save_to_csv>
X = df_feat
y = np.log(df['trip_duration'])
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42 )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.target = np.expm1(preds)
 sub.to_csv('torch.csv', index=False )<set_options>
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, random_state=42 )
%matplotlib inline<prepare_x_and_y>
rmse_rfr = []
def func_rfr(num):
rfr = RandomForestRegressor(n_estimators=num)
 rfr.fit(X_train, y_train)
 pred = rfr.predict(X_valid)
 rmse_rfr.append(np.sqrt(mean_squared_error(y_valid, pred)) )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 t = train_df.iloc[:,2:].duplicated(keep=False)
 duplicated_indices = t[t].index.values
print("Removed {} duplicated rows: {}".format(len(duplicated_indices), duplicated_indices))
train_df.iat[duplicated_indices[0], 1] = np.expm1(np.log1p(train_df.target.loc[duplicated_indices] ).mean())
 train_df.drop(duplicated_indices[1:], inplace=True)
 total_df = pd.concat([train_df.drop('target', axis=1), test_df], axis=0 ).drop('ID', axis=1)
 y = np.log1p(train_df.target)
 train_idx = range(0, len(train_df))
test_idx = range(len(train_df), len(total_df))
zero_std_cols = train_df.drop("ID", axis=1 ).columns[train_df.std() == 0]
total_df.drop(zero_std_cols, axis=1, inplace=True)
 print("Removed {} constant columns".format(len(zero_std_cols)))
 _, unique_indices = np.unique(total_df.iloc[train_idx], axis=1, return_index=True)
 colsToRemove = [i for i in range(total_df.shape[1])if i not in unique_indices]
total_df = total_df.iloc[:, unique_indices]
print("Dropped {} duplicate columns: {}".format(len(colsToRemove), colsToRemove))
total_df = np.log1p(total_df)
 for col in total_df.columns:
nonzero_rows = total_df[col] != 0
total_df.loc[nonzero_rows, col] = scale(total_df.loc[nonzero_rows, col].values)
<feature_engineering>
if __name__ == '__main__':
Thread(target = func_rfr(20)).start()
Thread(target = func_rfr(50)).start()
Thread(target = func_rfr(100)).start()
aggregate_df = pd.DataFrame()
n0_df = total_df[total_df>0]
aggregate_df['n0_count'] = total_df.astype(bool ).sum(axis=1)
 aggregate_df['n0_mean'] = n0_df.mean(axis=1)
 aggregate_df['n0_median'] = n0_df.median(axis=1)
 aggregate_df['n0_kurt'] = n0_df.kurt(axis=1)
 aggregate_df['n0_min'] = n0_df.min(axis=1)
 aggregate_df['n0_std'] = n0_df.std(axis=1)
 aggregate_df['n0_skew'] = n0_df.skew(axis=1)
 aggregate_df['mean'] = total_df.mean(axis=1)
 aggregate_df['std'] = total_df.std(axis=1)
 aggregate_df['max'] = total_df.max(axis=1)
 aggregate_df['nunique'] = total_df.nunique(axis=1)
 aggregate_df['sum_zeros'] =(total_df == 0 ).astype(int ).sum(axis=1)
 aggregate_df['geometric_mean'] = n0_df.apply(
 lambda x: np.exp(np.log(x ).mean()), axis=1 
)
 del n0_df
aggregate_df.reset_index(drop=True, inplace=True)
 print("Created features for: {}".format(aggregate_df.columns.values))<define_search_model>
rfr = RandomForestRegressor(n_estimators=100)
 rfr.fit(X_train, y_train)
 pred = rfr.predict(X_valid)
 score_rfr = cross_val_score(rfr, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error' )
COMPONENTS = 10
sparse_matrix = scipy.sparse.csr_matrix(total_df.values)
 methods = [
{'method': TSNE(n_components=3, init='pca'), 'data': 'train'},
{'method': TruncatedSVD(n_components=COMPONENTS), 'data': 'sparse'},
{'method': PCA(n_components=COMPONENTS), 'data': 'total'},
{'method': FastICA(n_components=COMPONENTS), 'data': 'total'},
{'method': GaussianRandomProjection(n_components=COMPONENTS, eps=0.1), 'data': 'total'},
{'method': SparseRandomProjection(n_components=COMPONENTS, dense_output=True), 'data': 'total'}
]
embeddings = []
for run in methods:
name = run['method'].__class__.__name__
if run['data'] == 'sparse':
embedding = run['method'].fit_transform(sparse_matrix)
 elif run['data'] == 'train':
embedding = run['method'].fit_transform(total_df.iloc[train_idx])
 else:
embedding = run['method'].fit_transform(total_df)
 embeddings.append(
 pd.DataFrame(embedding, columns=[f"{name}_{i}" for i in range(embedding.shape[1])])
)
 print(f">> Ran {name}")
 components_df = pd.concat(embeddings, axis=1 ).reset_index(drop=True )<choose_model_class>
print('cross_val_score average: ', abs(np.mean(score_rfr)))
 print('MAE: ',mean_absolute_error(y_valid, pred))
print('MSE: ',mean_squared_error(y_valid, pred))
print('RMSE: ',np.sqrt(mean_squared_error(y_valid, pred)) )
encoding_dim = 5
enc_input = Input(( total_df.shape[1],))
enc_output = Dense(128, activation='relu' )(enc_input)
 enc_output = Dropout(0.5 )(enc_output)
 enc_output = Dense(encoding_dim, activation='relu' )(enc_output)
 dec_input = Dense(128, activation='relu' )(enc_output)
 dec_output = Dropout(0.5 )(dec_input)
 dec_output = Dense(total_df.shape[1], activation='relu' )(dec_output)
 vanilla_encoder = Model(enc_input, enc_output)
 vanilla_autoencoder = Model(enc_input, dec_output)
 vanilla_autoencoder.compile(optimizer=Adam(0.0001), loss='mean_squared_error')
 vanilla_autoencoder.summary()
vanilla_autoencoder.fit(
 total_df.values, total_df.values,
epochs=6, batch_size=64,
shuffle=True 
)
 dense_ae_df = pd.DataFrame(
 vanilla_encoder.predict(total_df.values, batch_size=64),
columns=['dense_AE_{}'.format(i)for i in range(encoding_dim)]
).reset_index(drop=True )<define_variables>
pred = rfr.predict(test[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude', 'dropoff_longitude',
'dropoff_latitude', 'store_and_fwd_flag', 'travel_distance_km',
'hour_pickup', 'day_pickup']] )
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98']<statistical_test>
arr_id = test['id']
submission = pd.DataFrame({'id': arr_id, 'trip_duration': np.exp(pred)})
 print(submission)
 submission.to_csv("submit_file.csv", index=False )