Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<prepare_x_and_y>
%matplotlib inline
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(
(datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('
Time taken: %i hours %i minutes and %s seconds.' % 
(thour, tmin, round(tsec, 2)))
 def rmse_min_func(weights):
final_prediction = 0
for weight, prediction in zip(weights, blend_train):
final_prediction += weight * prediction
return np.sqrt(mean_squared_error(y_train, final_prediction))
start_time = timer(None)
 NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000
print('Reading in Data')
 df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 y_train = np.log1p(df_train['price'])
 tr_ids = df_train['train_id'].values.astype(np.int32)
 df_train.drop(['price', 'train_id'], axis=1, inplace=True)
 te_ids = df_test['test_id'].values.astype(np.int32)
 df_test.drop(['test_id'], axis=1, inplace=True)
 df = pd.concat([df_train, df_test])
 nrow_train = df_train.shape[0]
del df_train, df_test
gc.collect()
df['category_name'] = df['category_name'].fillna('Other' ).astype('category')
 df['brand_name'] = df['brand_name'].fillna('unknown')
 pop_brands = df['brand_name'].value_counts().index[:NUM_BRANDS]
df.loc[~df['brand_name'].isin(pop_brands), 'brand_name'] = 'Other'
df['item_description'] = df['item_description'].fillna('None')
 df['item_condition_id'] = df['item_condition_id'].astype('category')
 df['brand_name'] = df['brand_name'].astype('category')
 print('Encodings')
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df['name'])
 print('Category Encoders')
 unique_categories = pd.Series('/'.join(df['category_name'].unique().astype(
 'str')).split('/')).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df['category_name'])
 print('Descp encoders')
 count_descp = TfidfVectorizer(max_features=MAX_FEAT_DESCP,
ngram_range=(1, 3),
stop_words='english')
 X_descp = count_descp.fit_transform(df['item_description'])
 print('Brand encoders')
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df['brand_name'])
 print('Dummy Encoders')
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
'item_condition_id', 'shipping'
]], sparse=True ).values)
 X = scipy.sparse.hstack(( X_dummies, X_descp, X_brand, X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape, X_name.shape, X_descp.shape,
X_brand.shape])
 X_train = X[:nrow_train]
X_test = X[nrow_train:]
timer(start_time )
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<data_type_conversions>
folds = 3
sgd_cv_sum = 0
ridge_cv_sum = 0
lgb_cv_sum = 0
lgb_pred = []
sgd_pred = []
ridge_pred = []
lgb_fpred = []
sgd_fpred = []
ridge_fpred = []
avreal = y_train
lgb_avpred = np.zeros(X_train.shape[0])
 sgd_avpred = np.zeros(X_train.shape[0])
 ridge_avpred = np.zeros(X_train.shape[0])
 idpred = tr_ids
blend_train = []
blend_test = []
train_time = timer(None)
 kf = KFold(n_splits=folds, random_state=1001)
 for i,(train_index, val_index)in enumerate(kf.split(X_train, y_train)) :
start_time = timer(None)
 Xtrain, Xval = X_train[train_index], X_train[val_index]
ytrain, yval = y_train[train_index], y_train[val_index]
model = SGDRegressor(penalty='l2',
loss='squared_epsilon_insensitive',
max_iter=200,
tol=0.00001,
epsilon=0.0001,
learning_rate='invscaling',
fit_intercept=False,
alpha=1e-10,
l1_ratio=0.09,
shuffle=True,
verbose=0,
random_state=1001)
 model.fit(Xtrain, ytrain)
 sgd_scores_val = model.predict(Xval)
 sgd_RMSLE = np.sqrt(mean_squared_error(yval, sgd_scores_val))
print('
Fold %02d SGD RMSLE: %.6f' %(( i + 1), sgd_RMSLE))
sgd_y_pred = model.predict(X_test)
 model = Ridge(alpha=4.75,
solver='sag',
fit_intercept=False,
random_state=1001,
max_iter=1000)
 model.fit(Xtrain, ytrain)
 ridge_scores_val = model.predict(Xval)
 ridge_RMSLE = np.sqrt(mean_squared_error(yval, ridge_scores_val))
print(' Fold %02d Ridge RMSLE: %.6f' %(( i + 1), ridge_RMSLE))
ridge_y_pred = model.predict(X_test)
 params = {
'boosting': 'gbdt',
'max_depth': 7,
'min_data_in_leaf': 80,
'num_leaves': 40,
'learning_rate': 0.75,
'objective': 'regression',
'metric': 'rmse',
'nthread': 4,
'bagging_freq': 1,
'subsample': 0.94,
'colsample_bytree': 0.7,
'min_child_weight': 17,
'is_unbalance': False,
'verbose': -1,
'seed': 1001
}
dtrain = lgb.Dataset(Xtrain, label=ytrain, max_bin=4095)
 dval = lgb.Dataset(Xval, label=yval, max_bin=4095)
 watchlist = [dtrain, dval]
watchlist_names = ['train', 'val']
model = lgb.train(params,
train_set=dtrain,
num_boost_round=1800,
valid_sets=watchlist,
valid_names=watchlist_names,
early_stopping_rounds=100,
verbose_eval=50)
 lgb_scores_val = model.predict(Xval)
 lgb_RMSLE = np.sqrt(mean_squared_error(yval, lgb_scores_val))
print(' Fold %02d LightGBM RMSLE: %.6f' %(( i + 1), lgb_RMSLE))
lgb_y_pred = model.predict(X_test)
 del Xtrain, Xval
gc.collect()
timer(start_time)
 sgd_avpred[val_index] = sgd_scores_val
ridge_avpred[val_index] = ridge_scores_val
lgb_avpred[val_index] = lgb_scores_val
if i > 0:
sgd_fpred = sgd_pred + sgd_y_pred
ridge_fpred = ridge_pred + ridge_y_pred
lgb_fpred = lgb_pred + lgb_y_pred
else:
sgd_fpred = sgd_y_pred
ridge_fpred = ridge_y_pred
lgb_fpred = lgb_y_pred
sgd_pred = sgd_fpred
ridge_pred = ridge_fpred
lgb_pred = lgb_fpred
sgd_cv_sum = sgd_cv_sum + sgd_RMSLE
ridge_cv_sum = ridge_cv_sum + ridge_RMSLE
lgb_cv_sum = lgb_cv_sum + lgb_RMSLE
timer(train_time)
 sgd_cv_score =(sgd_cv_sum / folds)
 ridge_cv_score =(ridge_cv_sum / folds)
 lgb_cv_score =(lgb_cv_sum / folds)
 sgd_oof_RMSLE = np.sqrt(mean_squared_error(avreal, sgd_avpred))
ridge_oof_RMSLE = np.sqrt(mean_squared_error(avreal, ridge_avpred))
lgb_oof_RMSLE = np.sqrt(mean_squared_error(avreal, lgb_avpred))
print('
Average SGD RMSLE:\t%.6f' % sgd_cv_score)
 print(' Out-of-fold SGD RMSLE:\t%.6f' % sgd_oof_RMSLE)
 print('
Average Ridge RMSLE:\t%.6f' % ridge_cv_score)
 print(' Out-of-fold Ridge RMSLE:\t%.6f' % ridge_oof_RMSLE)
 print('
Average LightGBM RMSLE:\t%.6f' % lgb_cv_score)
 print(' Out-of-fold LightGBM RMSLE:\t%.6f' % lgb_oof_RMSLE)
 sgd_score = round(sgd_oof_RMSLE, 6)
 ridge_score = round(ridge_oof_RMSLE, 6)
 lgb_score = round(lgb_oof_RMSLE, 6)
 sgd_mpred = sgd_pred / folds
ridge_mpred = ridge_pred / folds
lgb_mpred = lgb_pred / folds
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['nb_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 df['ratio_pageviews'] = df['totals.pageviews'] / df['nb_pageviews']
y_reg = train['target']
del train['target']<define_variables>
blend_time = timer(None)
 blend_train.append(sgd_avpred)
 blend_train.append(ridge_avpred)
 blend_train.append(lgb_avpred)
 blend_train = np.array(blend_train)
 blend_test.append(sgd_mpred)
 blend_test.append(ridge_mpred)
 blend_test.append(lgb_mpred)
 blend_test = np.array(blend_test)
 print('
Finding Blending Weights...')
 res_list = []
weights_list = []
for k in range(20):
starting_values = np.random.uniform(size=len(blend_train))
bounds = [(-1, 1)] * len(blend_train)
 res = minimize(rmse_min_func,
starting_values,
method='L-BFGS-B',
bounds=bounds,
options={'disp': False,
'maxiter': 100000})
 res_list.append(res['fun'])
 weights_list.append(res['x'])
 print('{iter}\tScore: {score}\tWeights: {weights}'.format(
 iter=(k + 1),
score=res['fun'],
weights='\t'.join([str(item)for item in res['x']])))
 bestSC = np.min(res_list)
 bestWght = weights_list[np.argmin(res_list)]
weights = bestWght
blend_score = round(bestSC, 6)
 print('
Ensemble Score: {best_score}'.format(best_score=bestSC))
print('
Best Weights: {weights}'.format(weights=bestWght))
train_prices = np.zeros(len(blend_train[0]))
test_prices = np.zeros(len(blend_test[0]))
print('
Your final model:')
 for k in range(len(blend_test)) :
print(' %.6f * model-%d' %(weights[k],(k + 1)))
 test_prices += blend_test[k] * weights[k]
for k in range(len(blend_train)) :
train_prices += blend_train[k] * weights[k]
timer(blend_time )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
now = datetime.now()
oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(sgd_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_SGDRegressor-3fold-00-v1-oof_' + str(sgd_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(sgd_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your SGD Regressor prediction:')
 print(result.head(10))
sub_file = 'submission_SGDRegressor-3fold-00-v1_' + str(sgd_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(ridge_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_Ridge-3fold-00-v1-oof_' + str(ridge_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(ridge_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your Ridge Regressor prediction:')
 print(result.head(10))
sub_file = 'submission_Ridge-3fold-00-v1_' + str(ridge_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(lgb_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_LightGBM-3fold-00-v1-oof_' + str(lgb_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(lgb_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your LightGBM prediction:')
 print(result.head(10))
sub_file = 'submission_LightGBM-3fold-00-v1_' + str(lgb_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(train_prices)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_blend-3fold-00-v1-oof_' + str(blend_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(test_prices), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your blended prediction:')
 print(result.head(10))
sub_file = 'submission_blend-3fold-00-v1_' + str(blend_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')