Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<define_variables>
%matplotlib inline
plt.rcParams['figure.figsize'] = [16, 10]
days = env.get_prediction_days()<create_dataframe>
train = pd.read_csv('.. /input/new-york-city-taxi-with-osrm/train.csv')
 test = pd.read_csv('.. /input/new-york-city-taxi-with-osrm/test.csv')
 pd.set_option('display.float_format', lambda x: '%.3f' % x)
 train.head()
n_days = 0
predicted_confidences = np.array([])
 news_pre = deque()
news_all = pd.DataFrame()
BaseMod = 50
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 news_all = pd.concat([news_all,news_obs_df])
 if n_days >= BaseMod and n_days % BaseMod >= 0 and n_days % BaseMod < 8:
news_pre.append(news_obs_df)
 elif n_days >= BaseMod and n_days % BaseMod == 8:
del news_all
gc.collect()
news_all = pd.DataFrame()
for item in news_pre:
news_all = pd.concat([news_all,item])
 news_pre.clear()
news_all['time'] = news_all['time'].dt.floor('d')
 news_last = pd.DataFrame()
news_last = get_news_train(news_all)
 market_obs_df['key'] = market_obs_df['time'].astype(str)+ market_obs_df['assetName'].astype(str)
 market_obs_df = market_obs_df.join(news_last,on = 'key',how='left')
 market_obs_df['close'].clip(upper = 1000, inplace = True)
 market_obs_df['open'].clip(upper = 1000, inplace = True)
 market_obs_df['volume'].clip(upper = 1e+08, inplace = True)
 market_obs_df[num_cols]=market_obs_df[num_cols].fillna(col_mean)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_test = market_obs_df[num_cols]
X_test['assetCode'] = market_obs_df['assetCode'].apply(lambda x: encode(encoders[0], x)).values
X_test['assetName'] = market_obs_df['assetName'].apply(lambda x: encode(encoders[1], x)).values
market_prediction = model.predict(X_test)*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 del news_last
gc.collect()
env.write_submission_file()<set_options>
m = np.mean(train['trip_duration'])
 s = np.std(train['trip_duration'])
 train = train[train['trip_duration'] <= m + 2*s]
train = train[train['trip_duration'] >= m - 2*s]
train.head()
env = twosigmanews.make_env()<predict_on_test>
train = train[train['pickup_longitude'] <= -73.75]
train = train[train['pickup_longitude'] >= -74.03]
train = train[train['pickup_latitude'] <= 40.85]
train = train[train['pickup_latitude'] >= 40.63]
train = train[train['dropoff_longitude'] <= -73.75]
train = train[train['dropoff_longitude'] >= -74.03]
train = train[train['dropoff_latitude'] <= 40.85]
train = train[train['dropoff_latitude'] >= 40.63]
def predict(df):
df['predictMktres10'] = df.groupby(['assetCode'])['returnsOpenPrevMktres10'].shift(-11 ).fillna(0)
 df.loc[df.time > '2018-06-27', 'predictMktres10'] = 0
m = min(df[df.predictMktres10 > 0].predictMktres10.min() , df[df.predictMktres10 < 0].predictMktres10.max() * -1)
 df['assetcount'] = df.groupby('time' ).assetCode.transform('count' ).values
m1 = df.assetcount.min()
df['confidenceValue'] = 0
nz = df.predictMktres10 != 0
df.loc[nz, 'confidenceValue'] = m /(df[nz].predictMktres10)
 return df<predict_on_test>
train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)
 train.loc[:, 'pickup_date'] = train['pickup_datetime'].dt.date
test.loc[:, 'pickup_date'] = test['pickup_datetime'].dt.date
train['dropoff_datetime'] = pd.to_datetime(train.dropoff_datetime )
market_obs_df = None
pred_df = None
for(m_df, n_df, predictions_template_df)in env.get_prediction_days() :
env.predict(predictions_template_df)
 predictions_template_df['time'] = m_df.time.min()
if market_obs_df is None:
market_obs_df = m_df
pred_df = predictions_template_df
else:
market_obs_df = market_obs_df.append(m_df, ignore_index=True)
 pred_df = pred_df.append(predictions_template_df, ignore_index=True)
<data_type_conversions>
train.groupby('passenger_count' ).size()
pred_df.assetCode = pred_df.assetCode.astype(str)
 pred_df.time = pd.to_datetime(pred_df.time )<predict_on_test>
test.groupby('passenger_count' ).size()
t = predict(market_obs_df)
 t['returns'] = t.confidenceValue * t.predictMktres10
day_returns = t.groupby('time' ).returns.sum()
day_returns.mean() / day_returns.std()<merge>
def haversine_array(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
def dummy_manhattan_distance(lat1, lng1, lat2, lng2):
a = haversine_array(lat1, lng1, lat1, lng2)
 b = haversine_array(lat1, lng1, lat2, lng1)
 return a + b
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
pred_df.drop(columns=['confidenceValue'], inplace=True)
 pred_df = pred_df.merge(t[['time', 'assetCode', 'confidenceValue']], how='left', on=['time', 'assetCode'])\ 
.fillna(0)
 pred_df.time = pred_df.time.dt.date<save_to_csv>
train.loc[:, 'distance_haversine'] = haversine_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test.loc[:, 'distance_haversine'] = haversine_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 train.loc[:, 'distance_dummy_manhattan'] = dummy_manhattan_distance(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test.loc[:, 'distance_dummy_manhattan'] = dummy_manhattan_distance(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 train.loc[:, 'direction'] = bearing_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test.loc[:, 'direction'] = bearing_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values )
pred_df[['time', 'assetCode', 'confidenceValue']].to_csv('submission.csv', index=False, float_format='%.8f' )<categorify>
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values))
env = twosigmanews.make_env()
df = env._var07.set_index(["assetCode", "time"], verify_integrity=True)
 df["confidenceValue"] = 2 *(df.returnsOpenNextMktres10 > 0 ).astype(float)- 1
for(
 market_obs_df,
news_obs_df,
predictions_template_df, 
)in env.get_prediction_days() :
predictions_df =(
 predictions_template_df.set_index("assetCode")
.join(
 market_obs_df.set_index(["assetCode", "time"])
.join(df.confidenceValue, how="left")
.confidenceValue.reset_index() 
.drop("time", axis=1)
.set_index("assetCode"),
how="inner",
lsuffix="_", 
)
.drop("confidenceValue_", axis=1)
.reset_index() 
)
 env.predict(predictions_df)
 env.write_submission_file()<import_modules>
sample_ind = np.random.permutation(len(coords)) [:800000]
kmeans = MiniBatchKMeans(n_clusters=100, batch_size=10000 ).fit(coords[sample_ind] )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews<split>
train.loc[:, 'pickup_cluster'] = kmeans.predict(train[['pickup_latitude', 'pickup_longitude']])
 train.loc[:, 'dropoff_cluster'] = kmeans.predict(train[['dropoff_latitude', 'dropoff_longitude']])
 test.loc[:, 'pickup_cluster'] = kmeans.predict(test[['pickup_latitude', 'pickup_longitude']])
 test.loc[:, 'dropoff_cluster'] = kmeans.predict(test[['dropoff_latitude', 'dropoff_longitude']] )
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()<define_variables>
train['Month'] = train['pickup_datetime'].dt.month
test['Month'] = test['pickup_datetime'].dt.month
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10'] 
<split>
train.groupby('Month' ).size() ,test.groupby('Month' ).size()
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
<categorify>
train['DayofMonth'] = train['pickup_datetime'].dt.day
test['DayofMonth'] = test['pickup_datetime'].dt.day
len(train.groupby('DayofMonth' ).size()),len(test.groupby('DayofMonth' ).size() )
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders] 
<normalization>
train['Hour'] = train['pickup_datetime'].dt.hour
test['Hour'] = test['pickup_datetime'].dt.hour
len(train.groupby('Hour' ).size()),len(test.groupby('Hour' ).size() )
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
<choose_model_class>
train['dayofweek'] = train['pickup_datetime'].dt.dayofweek
test['dayofweek'] = test['pickup_datetime'].dt.dayofweek
len(train.groupby('dayofweek' ).size()),len(test.groupby('dayofweek' ).size() )
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(11,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<prepare_x_and_y>
fr1 = pd.read_csv('.. /input/new-york-city-taxi-with-osrm/fastest_routes_train_part_1.csv', usecols=['id', 'total_distance', 'total_travel_time', 'number_of_steps'])
 fr2 = pd.read_csv('.. /input/new-york-city-taxi-with-osrm/fastest_routes_train_part_2.csv', usecols=['id', 'total_distance', 'total_travel_time', 'number_of_steps'])
 test_street_info = pd.read_csv('.. /input/new-york-city-taxi-with-osrm/fastest_routes_test.csv',
usecols=['id', 'total_distance', 'total_travel_time', 'number_of_steps'])
 train_street_info = pd.concat(( fr1, fr2))
train = train.merge(train_street_info, how='left', on='id')
 test = test.merge(test_street_info, how='left', on='id' )
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<train_model>
vendor_train = pd.get_dummies(train['vendor_id'], prefix='vi', prefix_sep='_')
 vendor_test = pd.get_dummies(test['vendor_id'], prefix='vi', prefix_sep='_')
 passenger_count_train = pd.get_dummies(train['passenger_count'], prefix='pc', prefix_sep='_')
 passenger_count_test = pd.get_dummies(test['passenger_count'], prefix='pc', prefix_sep='_')
 store_and_fwd_flag_train = pd.get_dummies(train['store_and_fwd_flag'], prefix='sf', prefix_sep='_')
 store_and_fwd_flag_test = pd.get_dummies(test['store_and_fwd_flag'], prefix='sf', prefix_sep='_')
 cluster_pickup_train = pd.get_dummies(train['pickup_cluster'], prefix='p', prefix_sep='_')
 cluster_pickup_test = pd.get_dummies(test['pickup_cluster'], prefix='p', prefix_sep='_')
 cluster_dropoff_train = pd.get_dummies(train['dropoff_cluster'], prefix='d', prefix_sep='_')
 cluster_dropoff_test = pd.get_dummies(test['dropoff_cluster'], prefix='d', prefix_sep='_')
 month_train = pd.get_dummies(train['Month'], prefix='m', prefix_sep='_')
 month_test = pd.get_dummies(test['Month'], prefix='m', prefix_sep='_')
 dom_train = pd.get_dummies(train['DayofMonth'], prefix='dom', prefix_sep='_')
 dom_test = pd.get_dummies(test['DayofMonth'], prefix='dom', prefix_sep='_')
 hour_train = pd.get_dummies(train['Hour'], prefix='h', prefix_sep='_')
 hour_test = pd.get_dummies(test['Hour'], prefix='h', prefix_sep='_')
 dow_train = pd.get_dummies(train['dayofweek'], prefix='dow', prefix_sep='_')
 dow_test = pd.get_dummies(test['dayofweek'], prefix='dow', prefix_sep='_' )
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=2,
verbose=True,
callbacks=[early_stop,check_point] )<compute_test_metric>
passenger_count_test = passenger_count_test.drop('pc_9', axis = 1 )
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<define_variables>
train = train.drop(['id','vendor_id','passenger_count','store_and_fwd_flag','Month','DayofMonth','Hour','dayofweek',
'pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude'],axis = 1)
 Test_id = test['id']
test = test.drop(['id','vendor_id','passenger_count','store_and_fwd_flag','Month','DayofMonth','Hour','dayofweek',
'pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude'], axis = 1)
 train = train.drop(['dropoff_datetime','avg_speed_h','avg_speed_m','pickup_lat_bin','pickup_long_bin','trip_duration'], axis = 1 )
days = env.get_prediction_days()<categorify>
Train_Master = pd.concat([train,
vendor_train,
passenger_count_train,
store_and_fwd_flag_train,
cluster_pickup_train,
cluster_dropoff_train,
month_train,
dom_train,
hour_test,
dow_train
], axis=1 )
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test)[:,0]*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )<import_modules>
Test_master = pd.concat([test,
vendor_test,
passenger_count_test,
store_and_fwd_flag_test,
cluster_pickup_test,
cluster_dropoff_test,
month_test,
dom_test,
hour_test,
dow_test], axis=1 )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
Train_Master = Train_Master.drop(['pickup_datetime','pickup_date'],axis = 1)
 Test_master = Test_master.drop(['pickup_datetime','pickup_date'],axis = 1 )
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<set_options>
Train, Test = train_test_split(Train_Master[0:800000], test_size = 0.2 )
warnings.filterwarnings("ignore")
<define_variables>
X_train = Train.drop(['log_trip_duration'], axis=1)
 Y_train = Train["log_trip_duration"]
X_test = Test.drop(['log_trip_duration'], axis=1)
 Y_test = Test["log_trip_duration"]
Y_test = Y_test.reset_index().drop('index',axis = 1)
 Y_train = Y_train.reset_index().drop('index',axis = 1 )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
dtrain = xgb.DMatrix(X_train, label=Y_train)
 dvalid = xgb.DMatrix(X_test, label=Y_test)
 dtest = xgb.DMatrix(Test_master)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128<load_from_csv>
xgb_pars = {'min_child_weight': 1, 'eta': 0.09, 'colsample_bytree': 0.9,
'max_depth': 6,
'subsample': 0.9, 'lambda': 1., 'nthread': -1, 'booster' : 'gbtree', 'silent': 1,
'eval_metric': 'rmse', 'objective': 'reg:linear'}
model = xgb.train(xgb_pars, dtrain, 250, watchlist, early_stopping_rounds=250,
maximize=False, verbose_eval=1)
 print('Modeling RMSLE %.5f' % model.best_score )
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient" )<concatenate>
pred = model.predict(dtest)
 pred = np.exp(pred)- 1
<train_model><EOS> 
submission = pd.concat([Test_id, pd.DataFrame(pred)], axis=1)
 submission.columns = ['id','trip_duration']
submission['trip_duration'] = submission.apply(lambda x : 1 if(x['trip_duration'] <= 0)else x['trip_duration'], axis = 1)
 submission.to_csv("submission.csv", index=False )