Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<count_missing_values>
import gc, itertools
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
import xgboost as xgb
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
train.isnull().sum()<import_modules>
pd.options.display.max_rows = 10
from scipy import stats
from scipy.stats import norm<feature_engineering>
df_train = pd.read_csv('.. /input/sales_train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
train['trip_duration'] = np.log(train['trip_duration'].values )<concatenate>
df_items = pd.read_csv('.. /input/items.csv')
 df_categories = pd.read_csv('.. /input/item_categories.csv')
 df_shops = pd.read_csv('.. /input/shops.csv' )
feature_names=list(test)
 df_train=train[feature_names]
df=pd.concat(( df_train, test))<data_type_conversions>
df_train['datatype'], df_test['datatype'] = 'training', 'testing'
df_train.insert(0, 'ID', np.nan)
 df_test.insert(1, 'date', '01.11.2015')
 df_test.insert(2, 'date_block_num', 34)
 df_test.insert(df_test.shape[1] - 1, 'item_price', np.nan)
 df_test.insert(df_test.shape[1] - 1, 'item_cnt_day', np.nan)
 df_data = pd.concat([df_train, df_test], ignore_index=False )
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'] )<feature_engineering>
df_data.loc[df_data['shop_id'] == 0, 'shop_id'] = 57
df_data.loc[df_data['shop_id'] == 1, 'shop_id'] = 58
df_data.loc[df_data['shop_id'] == 11, 'shop_id'] = 10
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['weekday'] = df['pickup_datetime'].dt.weekday
df['hour'] = df['pickup_datetime'].dt.hour
df['dayofweek'] = df['pickup_datetime'].dt.dayofweek<drop_column>
df_data.loc[df_data['item_price'] < 0, 'item_price'] = df_data.loc[(df_data['date_block_num'] == 4)&(df_data['shop_id'] == 32)&(df_data['item_price'] > 0), 'item_price'].median()
df_data = df_data[(( df_data['item_price'] >= 0)&(df_data['item_price'] <= 100000)) |(df_data['item_price'].isna())]
df.drop(['pickup_datetime'], axis=1, inplace=True )<feature_engineering>
df_data = df_data[(df_data['item_cnt_day'] <= 1000)|(df_data['item_cnt_day'].isna())]
df['dist_long'] = df['pickup_longitude'] - df['dropoff_longitude']
df['dist_lat'] = df['pickup_latitude'] - df['dropoff_latitude']<feature_engineering>
df_block = df_data.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False ).agg({
'item_price': 'mean', 'item_cnt_day': sum
} ).rename(columns={'item_cnt_day': 'item_cnt_month'} )
df['dist'] = np.sqrt(np.square(df['dist_long'])+ np.square(df['dist_lat']))<compute_test_metric>
df_block = pd.merge(df_cross, df_block, how='left', left_on=['date_block_num', 'shop_id', 'item_id'], right_on=['date_block_num', 'shop_id', 'item_id'] )
def ft_haversine_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
df['distance'] = ft_haversine_distance(df['pickup_latitude'].values,
df['pickup_longitude'].values,
df['dropoff_latitude'].values,
df['dropoff_longitude'].values )<categorify>
df_block['item_cnt_month'] = df_block['item_cnt_month'].fillna(0 )
df = pd.concat([df, pd.get_dummies(df['store_and_fwd_flag'],prefix = 'store')], axis=1)
 df.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 df = pd.concat([df, pd.get_dummies(df['vendor_id'],prefix = 'vendor')], axis=1)
 df.drop(['vendor_id'], axis=1, inplace=True )<import_modules>
df_block = pd.merge(df_block, df_items, how='left', left_on='item_id', right_on='item_id' )
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_log_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import LinearRegression<drop_column>
df_block = pd.merge(df_block, df_categories, how='left', left_on='item_category_id', right_on='item_category_id' )
df.drop(["id"], axis=1, inplace=True )<split>
df_block = pd.merge(df_block, df_shops, how='left', left_on='shop_id', right_on='shop_id' )
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]<prepare_x_and_y>
df_block['item_id'].value_counts()
target = train['trip_duration']<split>
df_block['item_category_id'].value_counts()
X_train, X_val, y_train, y_val = train_test_split(new_train, target, test_size=0.2, shuffle=True )<compute_test_metric>
df_block['shop_id'].value_counts()
def rmsle_score(preds, true):
rmsle_score =(np.sum(( np.log1p(preds)-np.log1p(true)) **2)/len(true)) **0.5
return rmsle_score<import_modules>
df_items.loc[df_items['item_id'] == 20949, 'item_category_id']
RMSLE = make_scorer(rmsle_score )<import_modules>
df_items.loc[df_items['item_category_id'] == 71, 'item_id']
import statsmodels.api as sm<data_type_conversions>
df_block.loc[df_block['shop_id'] == 31, 'item_category_id'].value_counts()
model = sm.OLS(target.values, new_train.astype(float))<train_model>
df_block.loc[df_block['item_category_id'] == 40, 'item_id'].value_counts()
re = model.fit()
re.summary()<import_modules>
df_block.loc[df_block['item_category_id'] == 40, 'shop_id'].value_counts()
import lightgbm as lgbm<init_hyperparams>
df_block.loc[df_block['date_block_num'] == 33, 'item_id'][~df_block.loc[df_block['date_block_num'] == 33, 'item_id'].isin(df_block.loc[df_block['date_block_num'] < 33, 'item_id'])].value_counts()
lgb_params = {
'metric' : 'rmse',
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000 } 
<create_dataframe>
df_block.loc[df_block['date_block_num'] == 34, 'item_id'][~df_block.loc[df_block['date_block_num'] == 34, 'item_id'].isin(df_block.loc[df_block['date_block_num'] < 34, 'item_id'])].value_counts()
lgb_df = lgbm.Dataset(new_train,target )<train_model>
df_block.loc[df_block['date_block_num'] == 33, 'item_category_id'][~df_block.loc[df_block['date_block_num'] == 33, 'item_category_id'].isin(df_block.loc[df_block['date_block_num'] < 33, 'item_category_id'])].value_counts()
lgb_model = lgbm.train(lgb_params, lgb_df, num_boost_round=1500 )<predict_on_test>
df_block.loc[df_block['date_block_num'] == 34, 'item_category_id'][~df_block.loc[df_block['date_block_num'] == 34, 'item_category_id'].isin(df_block.loc[df_block['date_block_num'] < 34, 'item_category_id'])].value_counts()
pred = lgb_model.predict(new_test )<prepare_output>
df_block.loc[df_block['date_block_num'] == 33, 'shop_id'][~df_block.loc[df_block['date_block_num'] == 33, 'shop_id'].isin(df_block.loc[df_block['date_block_num'] < 33, 'shop_id'])].value_counts()
pred_lgb = np.exp(pred )<import_modules>
df_block.loc[df_block['date_block_num'] == 34, 'shop_id'][~df_block.loc[df_block['date_block_num'] == 34, 'shop_id'].isin(df_block.loc[df_block['date_block_num'] < 34, 'shop_id'])].value_counts()
import xgboost as xgb<init_hyperparams>
del df_categories, df_cross, df_data, df_items, df_shops
gc.collect()
params = {
'booster': 'gbtree',
'objective': 'reg:linear',
'learning_rate': 0.1,
'max_depth': 14,
'subsample': 0.8,
'colsample_bytree': 0.7,
'colsample_bylevel': 0.7,
'silent': 1
}<train_model>
df_block['year'] = df_block['date_block_num'] // 12
dtrain = xgb.DMatrix(new_train, target )<train_model>
df_block['month'] = df_block['date_block_num'] % 12
gbm = xgb.train(params,
dtrain,
num_boost_round = 200 )<predict_on_test>
day = pd.Series([31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31])
 df_block['day'] = df_block['month'].map(day )
pred_test = np.exp(gbm.predict(xgb.DMatrix(new_test)) )<prepare_output>
df_block.loc[df_block['shop_name'] == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
df_block['city'] = df_block['shop_name'].str.split(' ' ).apply(lambda x: x[0])
 df_block.loc[df_block['city'] == '!Якутск', 'city'] = 'Якутск'
df_block['city'].value_counts()
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )<prepare_output>
df_block['city_code'] = LabelEncoder().fit_transform(df_block['city'] )
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )<save_to_csv>
df_block['type'] = df_block['item_category_name'].str.split('-' ).apply(lambda x: x[0].strip())
 df_block['type'].value_counts()
sub.to_csv('submission.csv', index=False )<save_to_csv>
df_block['type_code'] = LabelEncoder().fit_transform(df_block['type'] )
sub.to_csv('submission.csv', index=False )<set_options>
df_block['subtype'] = df_block['item_category_name'].str.split('-' ).apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 df_block['subtype'].value_counts()
%matplotlib inline
warnings.filterwarnings("ignore" )<load_from_csv>
df_block['subtype_code'] = LabelEncoder().fit_transform(df_block['subtype'] )
train = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/train.zip')
 test = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/test.zip')
 sample_submission = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/sample_submission.zip' )<count_missing_values>
df_block = df_block.drop(['item_name', 'item_category_name', 'shop_name', 'city', 'type', 'subtype'], axis=1)
 gc.collect()
train.isnull().sum()<count_missing_values>
for i in [1, 2, 3, 4, 5, 6]:
shifted = df_block[['date_block_num', 'shop_id', 'item_id', 'item_price']].copy(deep=True)
 shifted.columns = ['date_block_num', 'shop_id','item_id', 'item_price_shift' + str(i)]
shifted['date_block_num'] = shifted['date_block_num'] + i
df_block = pd.merge(df_block, shifted, how='left', on=['date_block_num', 'shop_id', 'item_id'] )
train.isnull().sum()<import_modules>
df_block['item_price_by_item_mean'] = df_block.groupby(['item_id'])['item_price'].transform('mean' ).astype(np.float16 )
from scipy import stats
from scipy.stats import norm<feature_engineering>
for i in [1, 2, 3, 4, 5, 6]: df_block['item_price_by_item_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'item_id'])['item_price_shift' + str(i)].transform('mean' ).astype(np.float16 )
train['trip_duration'] = np.log(train['trip_duration'].values )<concatenate>
for i in [1, 2, 3, 4, 5, 6]: df_block['delta_item_price_by_item_date_mean_shift' + str(i)] =(df_block['item_price_by_item_date_mean_shift' + str(i)] - df_block['item_price_by_item_mean'])/ df_block['item_price_by_item_mean']
def select_nonnull(row):
for i in [1, 2, 3, 4, 5, 6]:
if not(np.isnan(row['delta_item_price_by_item_date_mean_shift' + str(i)])) : return row['delta_item_price_by_item_date_mean_shift' + str(i)]
return 0
df_block['delta_item_price_by_item_date_mean_shift'] = df_block.apply(select_nonnull, axis=1 )
feature_names=list(test)
 df_train=train[feature_names]
df=pd.concat(( df_train, test))<data_type_conversions>
df_block['item_cnt_month'] = df_block['item_cnt_month'].clip(0 ,20 )
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'] )<feature_engineering>
for i in [1, 2, 3, 6, 12]:
shifted = df_block[['date_block_num', 'shop_id', 'item_id', 'item_cnt_month']].copy(deep=True)
 shifted.columns = ['date_block_num', 'shop_id','item_id', 'item_cnt_month_shift' + str(i)]
shifted['date_block_num'] = shifted['date_block_num'] + i
df_block = pd.merge(df_block, shifted, how='left', on=['date_block_num', 'shop_id', 'item_id'] )
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['weekday'] = df['pickup_datetime'].dt.weekday
df['hour'] = df['pickup_datetime'].dt.hour
df['dayofweek'] = df['pickup_datetime'].dt.dayofweek<drop_column>
for i in [1]: df_block['item_cnt_month_by_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
df.drop(['pickup_datetime'], axis=1, inplace=True )<feature_engineering>
for i in [1, 2, 3, 6, 12]: df_block['item_cnt_month_by_item_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'item_id'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
df['dist_long'] = df['pickup_longitude'] - df['dropoff_longitude']
df['dist_lat'] = df['pickup_latitude'] - df['dropoff_latitude']<feature_engineering>
for i in [1, 2, 3, 6, 12]: df_block['item_cnt_month_by_category_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'item_category_id'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
df['dist'] = np.sqrt(np.square(df['dist_long'])+ np.square(df['dist_lat']))<compute_test_metric>
for i in [1, 2, 3, 6, 12]: df_block['item_cnt_month_by_shop_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'shop_id'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
def ft_haversine_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
df['distance'] = ft_haversine_distance(df['pickup_latitude'].values,
df['pickup_longitude'].values,
df['dropoff_latitude'].values,
df['dropoff_longitude'].values )<categorify>
for i in [1]: df_block['item_cnt_month_by_city_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'city_code'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
df = pd.concat([df, pd.get_dummies(df['store_and_fwd_flag'],prefix = 'store')], axis=1)
 df.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 df = pd.concat([df, pd.get_dummies(df['vendor_id'],prefix = 'vendor')], axis=1)
 df.drop(['vendor_id'], axis=1, inplace=True )<import_modules>
for i in [1]: df_block['item_cnt_month_by_item_city_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'item_id', 'city_code'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_log_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import LinearRegression<drop_column>
for i in [1]: df_block['item_cnt_month_by_category_shop_date_mean_shift' + str(i)] = df_block.groupby(['date_block_num', 'item_category_id', 'shop_id'])['item_cnt_month_shift' + str(i)].transform('mean' ).astype(np.float16 )
df.drop(["id"], axis=1, inplace=True )<split>
df_block['first_sale_item'] =(df_block['date_block_num'] - df_block.groupby(['item_id'])['date_block_num'].transform('min')).astype(np.int16)
 df_block['first_sale_item_shop'] =(df_block['date_block_num'] - df_block.groupby(['item_id', 'shop_id'])['date_block_num'].transform('min')).astype(np.int16 )
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]<prepare_x_and_y>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id'] )
target = train['trip_duration']<split>
for i in [12, 18, 24, 30]:
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] >= i)&(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_first_month' + str(i)+ 'onwards'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id'] )
X_train, X_val, y_train, y_val = train_test_split(new_train, target, test_size=0.2, shuffle=True )<compute_test_metric>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'shop_id'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_shop_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'shop_id'] )
def rmsle_score(preds, true):
rmsle_score =(np.sum(( np.log1p(preds)-np.log1p(true)) **2)/len(true)) **0.5
return rmsle_score<import_modules>
for i in [12, 18, 24, 30]:
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] >= i)&(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'shop_id'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_shop_first_month' + str(i)+ 'onwards'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'shop_id'] )
RMSLE = make_scorer(rmsle_score )<import_modules>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'city_code'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_city_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'city_code'] )
import statsmodels.api as sm<data_type_conversions>
for i in [12, 18, 24, 30]:
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] >= i)&(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'city_code'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_city_first_month' + str(i)+ 'onwards'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'city_code'] )
model = sm.OLS(target.values, new_train.astype(float))<train_model>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'type_code'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_type_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'type_code'] )
re = model.fit()
re.summary()<import_modules>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['item_category_id', 'subtype_code'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_category_subtype_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'item_category_id', 'subtype_code'] )
import lightgbm as lgbm<init_hyperparams>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['shop_id'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_shop_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'shop_id'] )
lgb_params = {
'metric' : 'rmse',
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000 } 
<create_dataframe>
list_of_first = []
for dateblocknum in df_block['date_block_num'].unique() :
df_first = df_block[(df_block['date_block_num'] < dateblocknum)&(df_block['first_sale_item'] == 0)].groupby(['city_code'], as_index=False ).agg({'item_cnt_month': 'mean'})
 df_first.insert(0, 'date_block_num', dateblocknum)
 list_of_first.append(df_first)
 df_first = pd.concat(list_of_first, ignore_index=True ).rename(columns={'item_cnt_month': 'item_cnt_month_by_city_first'})
 df_block = pd.merge(df_block, df_first, how='left', on=['date_block_num', 'city_code'] )
lgb_df = lgbm.Dataset(new_train,target )<train_model>
df_block = df_block[df_block['date_block_num'] > 11]
lgb_model = lgbm.train(lgb_params, lgb_df, num_boost_round=1500 )<predict_on_test>
col_fillnas = df_block.columns[df_block.isna().any() ].tolist()
df_block[col_fillnas] = df_block[col_fillnas].fillna(0 )
pred = lgb_model.predict(new_test )<prepare_output>
col_floats = [col for col in df_block.columns if df_block[col].dtypes == 'float64']
col_ints = [col for col in df_block.columns if df_block[col].dtypes == 'int64']
df_block[col_floats] = df_block[col_floats].astype(np.float16)
 df_block[col_ints] = df_block[col_ints].astype(np.int16)
 del df_first, list_of_first
gc.collect()
pred_lgb = np.exp(pred )<import_modules>
x = df_block[(df_block['first_sale_item'] != 0)&(df_block['date_block_num'] >= 30)&(df_block['date_block_num'] <= 33)].drop(['date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'item_category_id', 'year', 'month', 'day', 'city_code', 'type_code', 'subtype_code'] + [col for col in df_block.columns if col.startswith('item_price')], axis=1)
 y = df_block.loc[(df_block['first_sale_item'] != 0)&(df_block['date_block_num'] >= 30)&(df_block['date_block_num'] <= 33), 'item_cnt_month']
import xgboost as xgb<init_hyperparams>
x_train, x_validate, y_train, y_validate = train_test_split(x, y, random_state=58, test_size=0.25 )
params = {
'booster': 'gbtree',
'objective': 'reg:linear',
'learning_rate': 0.1,
'max_depth': 14,
'subsample': 0.8,
'colsample_bytree': 0.7,
'colsample_bylevel': 0.7,
'silent': 1
}<train_model>
del x, y
gc.collect()
dtrain = xgb.DMatrix(new_train, target )<train_model>
model_xgbreg = xgb.XGBRegressor(max_depth=8, learning_rate=0.3, n_estimators=500, objective='reg:linear', booster='gbtree', gamma=0.1, min_child_weight=300, subsample=0.8, colsample_bytree=0.8, reg_alpha=0, reg_lambda=1, random_state=58)
 model_xgbreg.fit(x_train, y_train, eval_set=[(x_train, y_train),(x_validate, y_validate)], early_stopping_rounds=10, verbose=False, callbacks=[xgb.callback.print_evaluation(period=10)])
 model_xgbreg_ypredict = model_xgbreg.predict(x_validate ).clip(0 ,20)
 model_xgbreg_rmse = mean_squared_error(y_validate, model_xgbreg_ypredict)** 0.5
print('xgboost regression
root mean squared error: %0.4f' %model_xgbreg_rmse )
gbm = xgb.train(params,
dtrain,
num_boost_round = 200 )<predict_on_test>
model_xgbreg_exist = model_xgbreg
pred_test = np.exp(gbm.predict(xgb.DMatrix(new_test)) )<prepare_output>
del x_train, x_validate, y_train, y_validate
gc.collect()
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )<prepare_output>
x = df_block[(df_block['first_sale_item'] == 0)&(df_block['date_block_num'] >= 30)&(df_block['date_block_num'] <= 33)].drop(['date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'item_category_id', 'year', 'month', 'day', 'city_code', 'type_code', 'subtype_code'] + [col for col in df_block.columns if col.startswith('item_price')], axis=1)
 y = df_block.loc[(df_block['first_sale_item'] == 0)&(df_block['date_block_num'] >= 30)&(df_block['date_block_num'] <= 33), 'item_cnt_month']
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )<save_to_csv>
x_train, x_validate, y_train, y_validate = train_test_split(x, y, test_size=0.25, random_state=58 )
sub.to_csv('submission.csv', index=False )<save_to_csv>
del x, y
gc.collect()
sub.to_csv('submission.csv', index=False )<load_from_csv>
model_xgbreg = xgb.XGBRegressor(max_depth=8, learning_rate=0.3, n_estimators=500, objective='reg:linear', booster='gbtree', gamma=0.1, min_child_weight=300, subsample=0.8, colsample_bytree=0.8, reg_alpha=0, reg_lambda=1, random_state=58)
 model_xgbreg.fit(x_train, y_train, eval_set=[(x_train, y_train),(x_validate, y_validate)], early_stopping_rounds=10, verbose=True)
 model_xgbreg_ypredict = model_xgbreg.predict(x_validate ).clip(0 ,20)
 model_xgbreg_rmse = mean_squared_error(y_validate, model_xgbreg_ypredict)** 0.5
print('xgboost regression
root mean squared error: %0.4f' %model_xgbreg_rmse )
test_data = pd.read_csv('.. /input/nyc-taxi-trip-duration/test.zip', index_col=0)
 train_data = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.zip', index_col=0)
 kaggle_submission = pd.read_csv('.. /input/nyc-taxi-trip-duration/sample_submission.zip' )<count_duplicates>
model_xgbreg_first = model_xgbreg
train_data.duplicated().sum()<remove_duplicates>
del x_train, x_validate, y_train, y_validate
gc.collect()
train_data = train_data.drop_duplicates()<count_missing_values>
df_test.loc[df_test['shop_id'] == 0, 'shop_id'] = 57
df_test.loc[df_test['shop_id'] == 1, 'shop_id'] = 58
df_test.loc[df_test['shop_id'] == 11, 'shop_id'] = 10
train_data.isna().sum()<filter>
final_model = [model_xgbreg_exist, model_xgbreg_first]
x_test = df_block[(df_block['first_sale_item'] != 0)&(df_block['date_block_num'] == 34)].drop(['date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'item_category_id', 'year', 'month', 'day', 'city_code', 'type_code', 'subtype_code'] + [col for col in df_block.columns if col.startswith('item_price')], axis=1)
 y_test = pd.DataFrame({'item_cnt_month': final_model[0].predict(x_test ).clip(0 ,20), 'shop_id': df_block.loc[(df_block['first_sale_item'] != 0)&(df_block['date_block_num'] == 34), 'shop_id'], 'item_id': df_block.loc[(df_block['first_sale_item'] != 0)&(df_block['date_block_num'] == 34), 'item_id']}, index=x_test.index)
 y_submit_exist = pd.merge(y_test, df_test[['ID', 'shop_id', 'item_id']], how='left', on=['shop_id', 'item_id'])
 x_test = df_block[(df_block['first_sale_item'] == 0)&(df_block['date_block_num'] == 34)].drop(['date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'item_category_id', 'year', 'month', 'day', 'city_code', 'type_code', 'subtype_code'] + [col for col in df_block.columns if col.startswith('item_price')], axis=1)
 y_test = pd.DataFrame({'item_cnt_month': final_model[1].predict(x_test ).clip(0 ,20), 'shop_id': df_block.loc[(df_block['first_sale_item'] == 0)&(df_block['date_block_num'] == 34), 'shop_id'], 'item_id': df_block.loc[(df_block['first_sale_item'] == 0)&(df_block['date_block_num'] == 34), 'item_id']}, index=x_test.index)
 y_submit_first = pd.merge(y_test, df_test[['ID', 'shop_id', 'item_id']], how='left', on=['shop_id', 'item_id'])
 y_submit = pd.concat([y_submit_exist, y_submit_first], ignore_index=True )
train_data = train_data[(train_data.trip_duration < 5000)]<filter>
out = pd.DataFrame({'ID': y_submit['ID'], 'item_cnt_month': y_submit['item_cnt_month']})
 out.to_csv('submission.csv', index=False )