Kaggle competition: Predict Future Sales
train_data = train_data[(train_data.passenger_count > 0)]<feature_engineering>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df
def add_date_features(data):
data['pickup_datetime'] = pd.to_datetime(data['pickup_datetime'])
 data['pickup_year'] = data['pickup_datetime'].dt.year
data['pickup_month'] = data['pickup_datetime'].dt.month
data['pickup_day'] = data['pickup_datetime'].dt.day
data['pickup_weekday'] = data['pickup_datetime'].dt.weekday
data['pickup_hour'] = data['pickup_datetime'].dt.hour
data['pickup_minutes'] = data['pickup_hour'] * 60 + data['pickup_datetime'].dt.minute
data['pickup_seconds'] = data['pickup_minutes'] * 60 + data['pickup_datetime'].dt.second
add_date_features(train_data)
 add_date_features(test_data )<categorify>
shape0 = sales_train.shape[0]
for col in ['item_price','item_cnt_day']:
max_val = sales_train[col].quantile (.99)
 sales_train = sales_train[sales_train[col]<max_val]
print(f'{shape0-sales_train.shape[0]} {col} values over {max_val} removed')
 print(f'new training set has {sales_train.shape[0]} records' )
train_data = pd.concat([train_data, pd.get_dummies(train_data['store_and_fwd_flag'])], axis=1)
 test_data = pd.concat([test_data, pd.get_dummies(test_data['store_and_fwd_flag'])], axis=1)
 train_data = pd.concat([train_data, pd.get_dummies(train_data['vendor_id'])], axis=1)
 test_data = pd.concat([test_data, pd.get_dummies(test_data['vendor_id'])], axis=1 )<compute_test_metric>
sales_train[sales_train['item_price']<=0]
def calculate_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
train_data['distance'] = calculate_distance(train_data['pickup_latitude'].values, train_data['pickup_longitude'].values, train_data['dropoff_latitude'].values, train_data['dropoff_longitude'].values)
 test_data['distance'] = calculate_distance(test_data['pickup_latitude'].values, test_data['pickup_longitude'].values, test_data['dropoff_latitude'].values, test_data['dropoff_longitude'].values )<filter>
sales_train=sales_train[sales_train['item_price']>0]
train_data = train_data[(train_data.distance < 200)]<feature_engineering>
test['date_block_num'] = 34
grid = pd.concat([grid, test[grid.columns]], ignore_index=True, sort=False)
 print('grid shape: ',grid.shape)
 print('missing values:',grid.isna().sum())
train_data['speed'] = train_data.distance / train_data.trip_duration<prepare_x_and_y>
index_cols = ['shop_id', 'item_id', 'date_block_num']
gb = sales_train.groupby(index_cols,as_index=False ).agg({'item_cnt_day':{'target':'sum','trips':'size'}})
 gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]
all_data = pd.merge(grid, gb, how='left', on=index_cols ).fillna(0)
 gb = sales_train.groupby(['shop_id', 'date_block_num'],as_index=False ).agg({'item_cnt_day':{'target_shop':'sum','trips_shop':'size'}})
 gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]
all_data = pd.merge(all_data, gb, how='left', on=['shop_id', 'date_block_num'] ).fillna(0)
 gb = sales_train.groupby(['item_id', 'date_block_num'],as_index=False ).agg({'item_cnt_day':{'target_item':'sum','trips_item':'size'}})
 gb.columns = [col[0] if col[-1] == '' else col[-1] for col in gb.columns.values]
all_data = pd.merge(all_data, gb, how='left', on=['item_id', 'date_block_num'] ).fillna(0)
 all_data.head()
columns = ['passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude',
'dropoff_latitude', 'pickup_month' , 'pickup_hour','pickup_minutes',
'pickup_seconds', 'pickup_weekday', 'pickup_day', 'distance']
X_train = train_data[columns]
y_train = train_data['trip_duration']<choose_model_class>
gb = sales_train.groupby(['date_block_num','item_id'],as_index=False ).agg({'item_price':{'median_item_price':'median'}})
 gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]
all_data = pd.merge(all_data, gb, how='left', on=['date_block_num','item_id'])
 print('na median_item_price:',all_data['median_item_price'].isna().sum())
rf = RandomForestRegressor()<train_model>
gb = all_data.groupby(['item_id'],as_index=False ).agg({'date_block_num':{'item_first_month':'min'}})
 gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]
all_data = pd.merge(all_data, gb, how='left', on=['item_id'] )
rf.fit(X_train, y_train )<predict_on_test>
all_data['new_item'] =(all_data['date_block_num']==all_data['item_first_month'] ).astype(int)
 all_data['new_item'].value_counts()
y_pred = rf.predict(X_test )<prepare_output>
all_data = downcast_dtypes(all_data)
 del grid, gb
gc.collect() ;
all_data.head()
kaggle_submission['trip_duration'] = y_pred
kaggle_submission.head()<save_to_csv>
cols_to_rename = list(all_data.columns.difference(index_cols+['item_first_month']))
shift_range = [1, 2, 3, 4, 5, 12]
for month_shift in tqdm_notebook(shift_range):
train_shift = all_data[index_cols + cols_to_rename].copy()
train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift
foo = lambda x: '{}_lag_{}'.format(x, month_shift)if x in cols_to_rename else x
train_shift = train_shift.rename(columns=foo)
 all_data = pd.merge(all_data, train_shift, on=index_cols, how='left' ).fillna(0)
 del train_shift
all_data = all_data[all_data['date_block_num'] >= 12]
fit_cols = [col for col in all_data.columns if col[-1] in [str(item)for item in shift_range]]
to_drop_cols = list(set(list(all_data.columns)) -(set(fit_cols)|set(index_cols)))+ ['date_block_num']
item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()
all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')
 all_data = downcast_dtypes(all_data)
 gc.collect() ;
kaggle_submission.to_csv('kaggle_submission.csv', index=False )<load_from_csv>
item_target_enc_na =.3343
cumsum = all_data.groupby('item_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_id')['target'].cumcount()
all_data['item_target_enc'] = cumsum/cumcnt
all_data['item_target_enc'].fillna(item_target_enc_na,inplace=True)
 corr = np.corrcoef(all_data['target'].values, all_data['item_target_enc'])[0][1]
print(corr )
train= pd.read_csv('.. /input/train.csv', index_col=0)
 test=pd.read_csv('.. /input/test.csv', index_col=0)
 print("Total number of samples in train file : ", train.shape[0])
 print("Total number of samples in test file : ", test.shape[0] )<filter>
item_target_enc_na =.3343
cumsum = all_data.groupby('shop_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('shop_id')['target'].cumcount()
all_data['shop_id_enc'] = cumsum/cumcnt
all_data['shop_id_enc'].fillna(item_target_enc_na,inplace=True)
 corr = np.corrcoef(all_data['target'].values, all_data['shop_id_enc'])[0][1]
print(corr )
train = train[(train.trip_duration < 5000)]<filter>
sales_train['month'] = sales_train['date_block_num']%12+1
sales_train[['date','month','date_block_num']].sample(5)
train = train.loc[(train['pickup_longitude'] > -75)&(train['pickup_longitude'] < -73)]
train = train.loc[(train['pickup_latitude'] > 40)&(train['pickup_latitude'] < 41)]<filter>
all_data['month'] = all_data['date_block_num']%12+1
train = train.loc[(train['dropoff_longitude'] > -75)&(train['dropoff_longitude'] < -73)]
train = train.loc[(train['dropoff_latitude'] > 40.5)&(train['dropoff_latitude'] < 41.5)]<filter>
all_data['item_category_id'] = all_data['item_id'].map(items.set_index('item_id')['item_category_id'] )
train = train.loc[(train['passenger_count'] >= 0)&(train['passenger_count'] <= 6)]<count_missing_values>
all_data['item_category_id'].isna().sum()
train.isnull().sum()<count_missing_values>
item_target_enc_na = 0
cumsum = all_data.groupby('item_category_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_category_id')['target'].cumcount()
item_category_id_enc = cumsum/cumcnt
item_category_id_enc.fillna(item_target_enc_na,inplace=True)
 all_data['item_category_id_enc'] = item_category_id_enc
corr = np.corrcoef(all_data['target'].values, item_category_id_enc)[0][1]
print(corr )
train.isnull().sum()<count_duplicates>
words = ' '.join(shops['shop_name'] ).split(' ')
 c = Counter(words)
 c.most_common(10 )
train.duplicated().sum()<count_duplicates>
shop_by_store = sales_train.groupby('shop_id',as_index=False)['item_cnt_day'].sum()
shop_by_store = shop_by_store.merge(shops, on='shop_id')
 print(shop_by_store['shop_name'].isna().sum())
 shop_by_store.head()
train = train.drop_duplicates()
train.duplicated().sum()<drop_column>
shop_by_store['name_array'] = shop_by_store['shop_name'].str.split(' ')
 top_words = [x for x,y in c.most_common(6)]
for w in top_words:
shop_by_store[w] = shop_by_store['shop_name'].map(lambda x: 1 if w in x else 0)
train.drop(["store_and_fwd_flag"], axis=1, inplace=True)
 test.drop(["store_and_fwd_flag"], axis=1, inplace=True )<categorify>
for w in top_words:
print(shop_by_store.groupby(by=w)['item_cnt_day'].mean() )
plg, plt = 'pickup_longitude', 'pickup_latitude'
dlg, dlt = 'dropoff_longitude', 'dropoff_latitude'
pdt, ddt = 'pickup_datetime', 'dropoff_datetime'<compute_test_metric>
top_words = top_words[0:4]
shops['name_array'] = shops['shop_name'].str.split(' ')
 for w in top_words:
shops[w] = shops['shop_name'].map(lambda x: 1 if w in x else 0)
 all_data = pd.merge(all_data,shops[['shop_id']+top_words],on='shop_id',how='left')
 print(all_data[top_words].isna().sum())
 all_data.head()
def haversine(lon1, lat1, lon2, lat2):

 lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat/2)**2 + cos(lat1)* cos(lat2)* sin(dlon/2)**2
c = 2 * asin(sqrt(a))
km = 6371* c
return km
def euclidian_distance(x):
x1, y1 = np.float64(x[plg]), np.float64(x[plt])
 x2, y2 = np.float64(x[dlg]), np.float64(x[dlt])
 return haversine(x1, y1, x2, y2 )<feature_engineering>
leaking_columns = ['median_item_price','date_block_num','target','target_shop','target_item','trips','trips_shop','trips_item']
X_train = all_data.loc[all_data['date_block_num'] < 33].drop(leaking_columns, axis=1)
 X_val = all_data.loc[all_data['date_block_num'] == 33].drop(leaking_columns, axis=1)
 X_test = all_data.loc[all_data['date_block_num'] == 34].drop(leaking_columns, axis=1)
 y_train = all_data.loc[all_data['date_block_num'] < 33,'target'].values
y_val = all_data.loc[all_data['date_block_num'] == 33,'target'].values
del all_data
gc.collect() ;
%time
train['distance'] = train[[plg, plt, dlg, dlt]].apply(euclidian_distance, axis=1 )<feature_engineering>
def validation_function(y_pred,y_true):
print(f'rmse before [0,20] clipping: {mean_squared_error(y_true, y_pred)}')
 y_pred = y_pred.clip(0,20)
 y_true = y_true.clip(0,20)
 print(f'rmse after [0,20] clipping: {mean_squared_error(y_true, y_pred)}')
 return
%time
test['distance'] = test[[plg, plt, dlg, dlt]].apply(euclidian_distance, axis=1 )<data_type_conversions>
lr = LinearRegression()
lr.fit(X_train.values, y_train)
 pred_lr = lr.predict(X_val.values)
 print('Test R-squared for linreg is %f' % r2_score(y_val, pred_lr))
validation_function(y_val,pred_lr )
train[pdt] = train[pdt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
train[ddt] = train[ddt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))<feature_engineering>
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'root_mean_squared_error'},
'num_leaves': 31,
'learning_rate': 0.05,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'verbose': 10
}
test[pdt] = test[pdt].apply(lambda x : datetime.strptime(x, "%Y-%m-%d %H:%M:%S")) 
<feature_engineering>
gbm = lgb.train(params,
lgb_train,
num_boost_round=15,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
early_stopping_rounds=5)
 print(gbm.pandas_categorical)
 lgb.plot_importance(gbm,figsize=(10,10)) ;
train['month'] = train[pdt].apply(lambda x : x.month)
 train['week_day'] = train[pdt].apply(lambda x : x.weekday())
 train['day_month'] = train[pdt].apply(lambda x : x.day)
 train['pickup_time_minutes'] = train[pdt].apply(lambda x : x.hour * 60.0 + x.minute )<feature_engineering>
categorical_features = ['shop_id','item_id','item_category_id','new_item']+top_words
lgb_train = lgb.Dataset(X_train, y_train,categorical_feature=categorical_features)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,categorical_feature=categorical_features)
 gbm2 = lgb.train(params,
lgb_train,
num_boost_round=15,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
categorical_feature = categorical_features,
early_stopping_rounds=5)
 print(gbm2.pandas_categorical)
 lgb.plot_importance(gbm2,figsize=(10,10)) ;
test['month'] = test[pdt].apply(lambda x : x.month)
 test['week_day'] = test[pdt].apply(lambda x : x.weekday())
 test['day_month'] = test[pdt].apply(lambda x : x.day)
 test['pickup_time_minutes'] = test[pdt].apply(lambda x : x.hour * 60.0 + x.minute )<prepare_x_and_y>
lgb_train = lgb.Dataset(X_train, y_train,categorical_feature=categorical_features)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,categorical_feature=categorical_features)
 lgb_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 2**7,
'bagging_freq':1,
'verbose':10
}
model = lgb.train(lgb_params,
lgb_train,
num_boost_round=1000,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
categorical_feature = categorical_features,
early_stopping_rounds=5)
 print(model.pandas_categorical)
 lgb.plot_importance(model,figsize=(10,10)) ;
features_train = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "distance", "month", "week_day", "day_month", "pickup_time_minutes"]
X_train = train[features_train]
y_train = np.log1p(train["trip_duration"])
 features_test = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "distance", "month", "week_day", "day_month", "pickup_time_minutes"]
X_test = test[features_test]<import_modules>
pred_lgb = model.predict(X_val)
 print('Test R-squared for LightGBM is %f' % r2_score(y_val, pred_lgb))
validation_function(y_val,pred_lgb )
from xgboost.sklearn import XGBRegressor
import scipy.stats as st
from sklearn.metrics import accuracy_score 
<choose_model_class>
X_train_level2 = np.c_[model.predict(X_train), lr.predict(X_train.values)]
lr2 = LinearRegression()
lr2.fit(X_train_level2, y_train)
 X_val_level2 = np.c_[model.predict(X_val), lr.predict(X_val.values)]
pred_lr2 = lr2.predict(X_val_level2)
 validation_function(y_val,pred_lr2)
reg=XGBRegressor(learning_rate=0.1, n_estimators=100, max_depth=50, n_jobs=1 )<train_model>
pred_lgb = model.predict(X_test ).clip(0,20)
 pred_lr = lr.predict(X_test.values ).clip(0,20)
 X_test_level2 = np.c_[pred_lgb, pred_lr]
pred_ensamble = lr2.predict(X_test_level2 )
reg.fit(X_train, y_train )<predict_on_test>
( test[['shop_id','item_id']].values == X_test[['shop_id','item_id']].values ).all()
log_pred = reg.predict(X_test)
 y_pred = np.exp(log_pred)- np.ones(len(log_pred)) 
<prepare_output>
pd.DataFrame(data={'ID':test['ID'],'item_cnt_month':pred_lgb} ).to_csv('lgbm_predictions.csv',index=False)
 pd.DataFrame(data={'ID':test['ID'],'item_cnt_month':pred_ensamble} ).to_csv('ensamble_predictions.csv',index=False)