Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.cluster import MiniBatchKMeans
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.regularizers import l2
env.write_submission_file()<import_modules>
train = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.csv')
 test = pd.read_csv('.. /input/nyc-taxi-trip-duration/test.csv' )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import datetime
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
import gc<split>
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 sample_ind = np.random.permutation(len(coords)) [:500000]
kmeans = MiniBatchKMeans(n_clusters=100, batch_size=10000 ).fit(coords[sample_ind])
 def toDateTime(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['pickup_weekday'] = df['pickup_datetime'].dt.weekday_name
df['pickup_day'] = df['pickup_datetime'].dt.day
df['pickup_month'] = df['pickup_datetime'].dt.month.astype('object')
 df['pickup_hour'] = df['pickup_datetime'].dt.hour
df['pickup_minute'] = df['pickup_datetime'].dt.minute
df['pickup_dt'] =(df['pickup_datetime'] - df['pickup_datetime'].min() ).map(
 lambda x: x.total_seconds())
 df.drop('pickup_datetime', axis = 1, inplace = True)
 return df
def haversine_np(lon1, lat1, lon2, lat2):
lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2.0)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2.0)**2
c = 2 * np.arcsin(np.sqrt(a))
km = 6367 * c
return km
def dummy_manhattan_distance(lat1, lng1, lat2, lng2):
a = haversine_np(lat1, lng1, lat1, lng2)
 b = haversine_np(lat1, lng1, lat2, lng1)
 return a + b
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
def locationFeatures(df):
df['up_town'] = np.sign(df['pickup_longitude'] - df['dropoff_longitude'])
 df['est_side'] = np.sign(df['pickup_latitude'] - df['dropoff_latitude'])
 df['haversine_distance'] = haversine_np(
 df['pickup_longitude'], df['pickup_latitude'],
df['dropoff_longitude'], df['dropoff_latitude'] 
)
 df['log_haversine_distance'] = np.log1p(df['haversine_distance'])
 df['distance_dummy_manhattan'] = dummy_manhattan_distance(
 df['pickup_latitude'], df['pickup_longitude'],
df['dropoff_latitude'], df['dropoff_longitude'] 
)
 df['log_distance_dummy_manhattan'] = np.log1p(df['distance_dummy_manhattan'])
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df.loc[:, 'pca_manhattan'] =(np.abs(df['dropoff_pca1'] - df['pickup_pca1'])+
np.abs(df['dropoff_pca0'] - df['pickup_pca0']))
df.loc[:, 'pickup_cluster'] = kmeans.predict(df[['pickup_latitude', 'pickup_longitude']] ).astype('object')
 df.loc[:, 'dropoff_cluster'] = kmeans.predict(df[['dropoff_latitude', 'dropoff_longitude']] ).astype('object')
 df.drop(['pickup_longitude', 'dropoff_longitude'], axis = 1, inplace = True)
 df.drop(['pickup_latitude', 'dropoff_latitude'], axis = 1, inplace = True)
 return df
def featureCreate(df):
print('Date time features')
 df = toDateTime(df)
 print('Location Features')
 df = locationFeatures(df)
 return df
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
gc.enable()<feature_engineering>
train = featureCreate(train)
 test = featureCreate(test)
 train['trip_duration'] = np.log1p(train['trip_duration'] )
news_train['time'] = news_train['time'].dt.floor('d')
 cols = ['sentimentNegative','sentimentNeutral','sentimentPositive','relevance','companyCount','bodySize','sentenceCount','wordCount','firstMentionSentence',
'sentimentWordCount','takeSequence','sentimentClass','noveltyCount12H', 'noveltyCount24H','noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D',
'volumeCounts12H','volumeCounts24H', 'volumeCounts3D', 'volumeCounts5D','volumeCounts7D']
news_total = news_train[['time','assetName'] + cols].copy()
del news_train
gc.collect()
news_train = news_total
print(news_train.columns)
<feature_engineering>
q1 = np.percentile(train['trip_duration'], 25)
 q3 = np.percentile(train['trip_duration'], 75)
 iqr = q3 - q1
train = train[ train['trip_duration'] <= q3 + 3.0*iqr]
train = train[ q1 - 3.0*iqr <= train['trip_duration']]
warnings.filterwarnings(action ='ignore',category = DeprecationWarning)
 def get_news_train(raw_data,days = 5):
news_last = pd.DataFrame()
rate = 1.0
for i in range(days):
cur_train = raw_data[cols] * rate
rate *= 0.7
cur_train['time'] = raw_data['time'] + datetime.timedelta(days = i,hours=22)
 cur_train['key'] = cur_train['time'].astype(str)+ raw_data['assetName'].astype(str)
 cur_train = cur_train[['key'] + cols].groupby('key' ).sum()
cur_train['key'] = cur_train.index.values
news_last = pd.concat([news_last, cur_train[['key'] + cols]])
 del cur_train
gc.collect()
print("after concat the shape is:",news_last.shape)
 news_last = news_last.groupby('key' ).sum()
news_last['key'] = news_last.index.values
print("the result shape is:",news_last.shape)
 del news_last['key']
return news_last
news_last = get_news_train(news_train)
 print(news_last.shape)
 print(news_last.head())
 print(news_last.dtypes)
<data_type_conversions>
labels = train.pop('trip_duration')
 train.drop(["id", "dropoff_datetime"], axis=1, inplace = True)
 sub = pd.DataFrame(columns = ['id', 'trip_duration'])
 sub['id'] = test.pop('id' )
market_train['key'] = market_train['time'].astype(str)+ market_train['assetName'].astype(str)
 market_train = market_train.join(news_last,on = 'key',how='left')
 print(market_train['sentimentNeutral'].isnull().value_counts())
 market_train.head()<define_variables>
train = pd.get_dummies(train)
 test = pd.get_dummies(test )
cat_cols = ['assetCode','assetName']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','sentimentNegative','sentimentNeutral','sentimentPositive','relevance','companyCount','bodySize',
'sentenceCount','wordCount','firstMentionSentence', 'sentimentWordCount','takeSequence','sentimentClass','noveltyCount12H',
'noveltyCount24H','noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H','volumeCounts24H', 'volumeCounts3D',
'volumeCounts5D','volumeCounts7D'] 
<split>
mmScale = MinMaxScaler()
n = train.shape[1]
train = mmScale.fit_transform(train )
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23 )<categorify>
model = Sequential()
model.add(Dense(n, activation='relu', input_shape=(n,)))
 model.add(Dense(int(0.5*n), activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mse', optimizer='rmsprop', metrics=['mae'] )
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for i in range(len(cat_cols)) ]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]<prepare_x_and_y>
print('On to the next one')
 model.fit(train, labels.values, epochs = 3)
 print('Finished' )
def get_input(market_train, indices):
X = market_train.loc[indices, num_cols]
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices)
 X_train.shape
print(X_valid.shape )<choose_model_class>
train_pred = model.predict(train)
 test_pred = model.predict(mmScale.transform(test))
<predict_on_test><EOS> 
sub['trip_duration'] = np.expm1(test_pred)
 sub.to_csv('submission111.csv', index=False )