Kaggle competition: New York City Taxi Trip Duration
def write_submission(model, env):
days = env.get_prediction_days()
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['date'] = market_obs_df.time.dt.date
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time']]
preds = model.predict_proba(obs_df[feats])[:, 1] * 2 - 1
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
write_submission(clf, env )<set_options>
%matplotlib inline
env = twosigmanews.make_env()<prepare_x_and_y>
df = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample = pd.read_csv('.. /input/sample_submission.csv' )
( market_train_df, news_train_df)= env.get_training_data()<count_missing_values>
is_dup = df.duplicated().sum()
is_null = df.isna().sum()
print("null : "+ str(is_null[::][1].sum()))
print("duplicated : "+ str(is_dup))
print(" ---------------- Before removing missing values")
 print(market_train_df.isna().sum())
 market_train_df.dropna(inplace= True)
 print("
---------------- After removing missing values ")
 print(market_train_df.isna().sum() )<filter>
df = df.loc[df['pickup_longitude']> -74.02]
df = df.loc[df['pickup_latitude']< 40.77]
market_train_df_1 = market_train_df[market_train_df.time.dt.year >= 2013]<count_values>
df = df.loc[df['dropoff_longitude']> -74.02]
df = df.loc[df['dropoff_latitude']< 40.77]
print("No.of observations(2013 to 2016): ", market_train_df_1.shape[0])
 print(market_train_df_1.time.dt.year.value_counts() )<set_options>
df = df[df['passenger_count'] > 0]
df = df[df['passenger_count'] <= 6]
del market_train_df
gc.collect()<filter>
df['log_trip_duration'] = np.log(df['trip_duration'].values )
news_train_df_1 = news_train_df[news_train_df.time.dt.year >= 2013].copy()<set_options>
df = df[(df['log_trip_duration'] > 3.5)]
df = df[(df['log_trip_duration'] < 9)]
del news_train_df
gc.collect()<count_values>
df['dist'] = np.sqrt(np.square(df['pickup_longitude'] - df['dropoff_longitude'])+ np.square(df['pickup_latitude'] - df['dropoff_latitude']))
df_test['dist'] = np.sqrt(np.square(df_test['pickup_longitude'] - df_test['dropoff_longitude'])+ np.square(df_test['pickup_latitude'] - df_test['dropoff_latitude']))
news_train_df_1.time.dt.year.value_counts()<define_variables>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )
news_var = ['time','assetName', 'bodySize','companyCount','sentenceCount','wordCount',
'firstMentionSentence','relevance','sentimentClass','sentimentNegative',
'sentimentNeutral','sentimentPositive','sentimentWordCount','noveltyCount12H',
'noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D',
'volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D'
]<feature_engineering>
df['m'] = df['pickup_datetime'].dt.month
df_test['m'] = df_test['pickup_datetime'].dt.month
df['d'] = df['pickup_datetime'].dt.dayofweek
df_test['d'] = df_test['pickup_datetime'].dt.dayofweek
df['h'] = df['pickup_datetime'].dt.hour
df_test['h'] = df_test['pickup_datetime'].dt.hour
news_train_df_1 = news_train_df_1[news_var]
news_train_df_1['date'] = news_train_df_1.time.dt.date<groupby>
X_train = df[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","m","h","d"]]
X_test = df_test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","m","h","d"]]
y = df["log_trip_duration"]
news_train_df_grp = news_train_df_1.groupby(['date','assetName'] ).mean().reset_index()<merge>
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
import lightgbm as lgb
market_train_df_1['date'] = market_train_df_1.time.dt.date
market_train_df_1 = pd.merge(market_train_df_1,news_train_df_grp,how='left',on = ['assetName','date'])
 market_train_df_1.head()<count_missing_values>
xtrain, xvalid, ytrain, yvalid = train_test_split(X_train,y, test_size=0.2, random_state=1337)
 xtrain.shape, xvalid.shape, xtrain.shape, yvalid.shape
market_train_df_1.isna().sum() /market_train_df_1.shape[0]<set_options>
dtrain = lgb.Dataset(X_train,y )
del news_train_df_grp
gc.collect()<drop_column>
lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'num_leaves': 1000,
'objective': 'regression',
'max_bin': 1000}
market_train_df_1 = market_train_df_1[market_train_df_1.universe == 1].copy()<define_variables>
cv_results = lgb.cv(lgb_params,dtrain,num_boost_round=100,nfold=3,metrics='mae',early_stopping_rounds=10,stratified=False )
id_train = market_train_df_1.time.dt.year != 2016
id_test = market_train_df_1.time.dt.year == 2016
dep_var = 'returnsOpenNextMktres10'
ind_var = ['volume', 'close', 'open', 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','bodySize', 'companyCount', 'sentenceCount', 'wordCount',
'firstMentionSentence', 'relevance', 'sentimentClass',
'sentimentNegative', 'sentimentNeutral', 'sentimentPositive',
'sentimentWordCount', 'noveltyCount12H', 'noveltyCount24H',
'noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H',
'volumeCounts24H', 'volumeCounts3D', 'volumeCounts5D',
'volumeCounts7D']
df_train = market_train_df_1.loc[id_train,ind_var]
df_test = market_train_df_1.loc[id_test,ind_var]
print("{0} training rows and {1} testing rows".format(df_train.shape[0],df_test.shape[0]))
y_train = market_train_df_1.loc[id_train,dep_var]
y_test = market_train_df_1.loc[id_test,dep_var]<train_model>
print('score :', cv_results['l1-mean'][-1] )
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
max_depth = 5, alpha = 10, n_estimators = 100)
 xg_reg.fit(df_train,y_train )<compute_train_metric>
model_lgb = lgb.train(lgb_params,dtrain,num_boost_round=1200 )
pred_train = xg_reg.predict(df_train)
 rms_train = sqrt(mean_squared_error(y_train, pred_train))
pred_test = xg_reg.predict(df_test)
 rms_test = sqrt(mean_squared_error(y_test, pred_test))
print('Train RMSE: {0} Test RMSE: {1}'.format(rms_train,rms_test))<feature_engineering>
pred = np.exp(model_lgb.predict(X_test))
pred
pred_test_df = market_train_df_1.loc[id_test,['time','assetCode','universe','returnsOpenNextMktres10']]
pred_test_df['dayofyear'] = pred_test_df.time.dt.dayofyear
pred_test_df['confidence'] = [1 if pred >=0 else -1 for pred in pred_test]
pred_test_df['score'] = pred_test_df.universe * pred_test_df.returnsOpenNextMktres10 * pred_test_df.confidence
print(pred_test_df.confidence.value_counts())
 score_1 = pred_test_df.groupby(['dayofyear'] ).score.sum()
score_2 = score_1.mean() / score_1.std()
print("
Competition Score: ",np.round(score_2,4))<define_variables>
df_sample['trip_duration'] = pred
submit = df_sample
submit.head(10 )
days = env.get_prediction_days()<merge>
submit_file = pd.DataFrame({"id": df_test.id, "trip_duration": pred} )
<predict_on_test><EOS> 
submit_file.to_csv('submission.csv', index=False )