Kaggle competition: Mercari Price Suggestion Challenge
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<prepare_x_and_y>
import pandas as pd
import numpy as np
import scipy
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
import lightgbm as lgb
import gc
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000
print("Reading in Data")
 df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df = pd.concat([df_train, df_test], 0)
 nrow_train = df_train.shape[0]
y_train = np.log1p(df_train["price"])
 del df_train
gc.collect()
print(df.memory_usage(deep = True))
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["item_description"] = df["item_description"].fillna("None")
 df["item_condition_id"] = df["item_condition_id"].astype("category")
 df["brand_name"] = df["brand_name"].astype("category")
 print(df.memory_usage(deep = True))
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
print("Encodings")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 print("Category Encoders")
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 print("Descp encoders")
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 print("Brand encoders")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 print("Dummy Encoders")
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape,
X_name.shape, X_descp.shape, X_brand.shape])
 X_train = X[:nrow_train]
X_test = X[nrow_train:]
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
params = {
'learning_rate': 0.75,
'application': 'regression',
'max_depth': 3,
'num_leaves': 100,
'verbosity': -1,
'metric': 'RMSE',
}
train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train, test_size = 0.1, random_state = 144)
 d_train = lgb.Dataset(train_X, label=train_y, max_bin=8192)
 d_valid = lgb.Dataset(valid_X, label=valid_y, max_bin=8192)
 watchlist = [d_train, d_valid]
model = lgb.train(params, train_set=d_train, num_boost_round=2200, valid_sets=watchlist, \
early_stopping_rounds=50, verbose_eval=100)
 preds = model.predict(X_test)
 model = Ridge(solver = "lsqr", fit_intercept=False)
 print("Fitting Model")
 model.fit(X_train, y_train)
 preds += model.predict(X_test)
 preds /= 2
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
df_test["price"] = np.expm1(preds)
 df_test[["test_id", "price"]].to_csv("submission_LGBM_Ridge_3.csv", index = False )
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 10000
print("Reading in Data")
 df_train2 = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test2 = pd.read_csv('.. /input/test.tsv', sep='\t')
 df = pd.concat([df_train2, df_test2], 0)
 nrow_train = df_train2.shape[0]
y_train = np.log(df_train2["price"]+1)
 del df_train2
gc.collect()
print(df.memory_usage(deep = True))
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["item_description"] = df["item_description"].fillna("None")
 df["item_condition_id"] = df["item_condition_id"].astype("category")
 df["brand_name"] = df["brand_name"].astype("category")
 print(df.memory_usage(deep = True))
print("Encodings")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 print("Category Encoders")
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 print("Descp encoders")
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 print("Brand encoders")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 print("Dummy Encoders")
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values)
 X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape,
X_name.shape, X_descp.shape, X_brand.shape])
 X_train = X[:nrow_train]
model = Ridge(solver = "lsqr", fit_intercept=False)
 print("Fitting Model")
 model.fit(X_train, y_train)
 X_test = X[nrow_train:]
preds1 = model.predict(X_test)
 df_test2["price1"] = np.exp(preds1)-1
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<prepare_x_and_y>
X=X_train
y=y_train
def rmsle(predictions, targets):
predictions = np.exp(predictions)- 1
targets = np.exp(targets)- 1
return np.sqrt(((predictions - targets)** 2 ).mean())
 def rmsle_lgb(labels, preds):
return 'rmsle', rmsle(preds, labels), False
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
print('Training model...')
 X_train, X_test2, y_train, y_test2 = train_test_split(X, y, test_size=0.3, random_state=42)
 lgbm_params = {'n_estimators': 1000, 'learning_rate': 0.4, 'max_depth': 10,
'num_leaves': 31, 'subsample': 0.9, 'colsample_bytree': 0.8,
'min_child_samples': 50, 'n_jobs': 4}
model = LGBMRegressor(**lgbm_params)
 model.fit(X_train, y_train,
eval_set=[(X_test2, y_test2)],
eval_metric=rmsle_lgb,
early_stopping_rounds=100,
verbose=True)
 print('Generating submission...')
 preds2 = model.predict(X_test)
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
df_test2['price2']=np.exp(preds2)- 1
df_test2['price']=(df_test2['price2']+df_test2['price1'])/2
df_test2[["test_id", "price"]].to_csv("submission.csv", index = False)
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
new = df_test[["test_id", "price"]].copy()
new.head()
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape<prepare_x_and_y>
new2 = df_test2[["test_id", "price"]].copy()
new2.head()
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
new['price'] = new2['price']*.6 + new['price']*.4
new.head()
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<categorify>
new.to_csv("lgbm_ridge_ensemble2.csv", float_format='%.4f', index=None )