Kaggle competition: New York City Taxi Trip Duration
y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.head()<load_from_csv>
%matplotlib inline
sns.set({'figure.figsize':(16,8)} )
sub_elastic = pd.read_csv('.. /input/houseprice-elasticnet-finish/submission.csv')
 sub_elastic.head()<save_to_csv>
train = pd.read_csv(".. /input/nyc-taxi-duration-eda-by-nguyen-khac-bao-anh/training_data.csv")
 test = pd.read_csv(".. /input/nyc-taxi-duration-eda-by-nguyen-khac-bao-anh/testing_data.csv" )
result =(sub.SalePrice + sub_elastic.SalePrice)/ 2
sub['SalePrice'] = result
sub.to_csv('submission_3_ensemble.csv', index=False)
 sub.head()<set_options>
xtrain = train.drop(['id', 'pickup_datetime', 'dropoff_datetime', 'trip_duration', 'log_trip_duration'], axis = 1 ).as_matrix()
xtest = test.drop(['id', 'pickup_datetime', ], axis = 1 ).as_matrix()
y = train['log_trip_duration'].values
del(train, test )
pd.set_option('max_columns', 50 )<split>
from sklearn.model_selection import train_test_split, cross_val_score
env = twosigmanews.make_env()
market_train, news_train = env.get_training_data()<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(xtrain,y, test_size=0.2, random_state=42)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape
start = datetime(2009, 1, 1, 0, 0, 0 ).date()
market_train = market_train.loc[market_train['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )<feature_engineering>
def rmse(y,pred):
return np.sqrt(np.mean(np.square(np.log(np.exp(y)) -np.log(np.exp(pred)))) )
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train
news_train = preprocess_news(news_train )<drop_column>
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df
index_df = unstack_asset_codes(news_train)
 index_df.head()<merge>
import lightgbm as lgb
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )<data_type_conversions>
dtrain = lgb.Dataset(xtrain,y)
 del(X_train, y_train, X_valid, y_valid, xtrain,y )
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols)
 news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )<merge>
lgb_params = {
'learning_rate': 0.1,
'max_depth': 8,
'num_leaves': 55,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 300}
market_train['date'] = market_train.time.dt.date
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )<compute_test_metric>
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )<data_type_conversions>
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()<split>
model_lgb = lgb.train(lgb_params,
dtrain,
num_boost_round=1500 )
train_index, test_index = train_test_split(df.index.values, test_size=0.1, shuffle=False )<choose_model_class>
submit = pd.read_csv('.. /input/nyc-taxi-trip-duration/sample_submission.csv')
 submit.head()
def evaluate_model(df, target, train_index, test_index, params):
params['n_jobs'] = 2
model = LGBMClassifier(**params)
 model.fit(df.iloc[train_index], target.iloc[train_index])
 return log_loss(target.iloc[test_index], model.predict_proba(df.iloc[test_index]))<define_search_space>
pred_test = np.exp(model_lgb.predict(xtest))
param_grid = {
'learning_rate': [0.15, 0.1, 0.05, 0.02, 0.01],
'num_leaves': [i for i in range(12, 90, 6)],
'n_estimators': [50, 200, 400, 600, 800],
'min_child_samples': [i for i in range(10, 100, 10)],
'colsample_bytree': [0.8, 0.9, 0.95, 1],
'subsample': [0.8, 0.9, 0.95, 1],
'reg_alpha': [0.1, 0.2, 0.4, 0.6, 0.8],
'reg_lambda': [0.1, 0.2, 0.4, 0.6, 0.8],
}
best_eval_score = 0
for i in range(100):
params = {k: np.random.choice(v)for k, v in param_grid.items() }
score = evaluate_model(df, bin_target, train_index, test_index, params)
 if score < best_eval_score or best_eval_score == 0:
best_eval_score = score
best_params = params
print("Best evaluation logloss", best_eval_score )<train_model>
submit['trip_duration'] = pred_test
clf = LGBMClassifier(**best_params)
 clf.fit(df, bin_target )<merge>
submit.to_csv("submit_file.csv", index=False )