Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<define_variables>
start_time = time.time()
df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df_train[df_train.price >= 3.0]
train_rows = len(df_train)
 all_data = pd.concat([df_train, df_test])
 print("[{}] Data concatenated.".format(time.time() - start_time))
def transform_values(data):
data.fillna(value='missing', inplace = True)
 return data
all_data = transform_values(all_data)
 def transform_category_name(category_name):
try:
main, sub1, sub2= category_name.split('/')
 return main, sub1, sub2
except:
return 'missing', 'missing', 'missing'
all_data['category_main'], all_data['category_sub1'], all_data['category_sub2'] = zip(*all_data['category_name'].apply(transform_category_name))
all_data[all_data.item_description != 'missing']
print("[{}] Data transformed.".format(time.time() - start_time))
name_cv = CountVectorizer(ngram_range=(1,2), min_df=7, stop_words='english')
 X_name = name_cv.fit_transform(all_data['name'])
 print("[{}] Name vector created.".format(time.time() - start_time))
cat_cv = CountVectorizer()
X_catmain = cat_cv.fit_transform(all_data['category_main'],)
 X_catsub1 = cat_cv.fit_transform(all_data['category_sub1'])
 X_catsub2 = cat_cv.fit_transform(all_data['category_sub2'])
 print("[{}] Category vectors created.".format(time.time() - start_time))
def descrLength(description):
try:
if description == "missing":
return 0
else:
words = [w for w in description.split(" ")]
return len(words)
 except:
return 0
all_data['descr_length'] = all_data['item_description'].apply(lambda x: descrLength(x))
all_data['descr_length'] = all_data['descr_length'].astype(str)
 X_length = cat_cv.fit_transform(all_data['descr_length'])
 print("[{}] Description length vector created.".format(time.time() - start_time))
vectorizer = TfidfVectorizer(ngram_range=(1,2), max_features=90000, strip_accents="ascii",
stop_words='english')
 X_description = vectorizer.fit_transform(all_data['item_description'])
 print("[{}] Description vector created.".format(time.time() - start_time))
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(all_data['brand_name'])
 print("[{}] Brand vector created.".format(time.time() - start_time))
X_dummies = csr_matrix(pd.get_dummies(all_data[['item_condition_id', 'shipping']],
sparse=True ).values)
 print("[{}] Shipping and condition vector created.".format(time.time() - start_time))
train_X = hstack(( X_name[:train_rows], X_catmain[:train_rows], X_catsub1[:train_rows],
X_catsub2[:train_rows], X_brand[:train_rows], X_length[:train_rows], X_description[:train_rows],
X_dummies[:train_rows]))
print('[{}] Training data stacked'.format(time.time() - start_time))
test_X = hstack(( X_name[train_rows:], X_catmain[train_rows:], X_catsub1[train_rows:],
X_catsub2[train_rows:], X_brand[train_rows:], X_length[train_rows:], X_description[train_rows:],
X_dummies[train_rows:]))
print('[{}] Test data stacked.'.format(time.time() - start_time))
Y_train = np.log1p(df_train['price'])
 print("[{}] Data processing and feature extraction complete.".format(time.time() - start_time))
feats = [c for c in df.columns if c not in removed_cols]<import_modules>
ridge_time = time.time()
data_train_X, data_val_X, data_train_Y, data_val_Y = train_test_split(train_X, Y_train)
 print('[{}] Data splitted.'.format(time.time() - ridge_time))
clf1 = Ridge(alpha=5.3, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=None)
 clf1.fit(train_X, Y_train)
 print("[{}] Data 1 fitted.".format(time.time() - ridge_time))
clf2 = Ridge(alpha=.75, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=100)
 clf2.fit(train_X, Y_train)
 print("[{}] Data 2 fitted.".format(time.time() - ridge_time))
from sklearn.tree import DecisionTreeRegressor<choose_model_class>
lgb_time = time.time()
data_train_X, data_val_X, data_train_Y, data_val_Y = train_test_split(train_X, Y_train)
 print('[{}] Data splitted.'.format(time.time() - lgb_time))
dataset_train = lgb.Dataset(data_train_X, label=data_train_Y)
 dataset_val = lgb.Dataset(data_val_X, label=data_val_Y)
 watchlist = [dataset_train, dataset_val]
params = {
'learning_rate': 0.35,
'application': 'regression',
'max_depth': 6,
'num_leaves': 60,
'verbosity': -1,
'metric': 'RMSE',
'data_random_seed': 1,
'bagging_fraction': 0.5,
'nthread': 4
}
lgb_model = lgb.train(params, train_set=dataset_train, num_boost_round=7500, valid_sets=watchlist,
early_stopping_rounds=1000, verbose_eval=500)
 print("[{}] Model completed.".format(time.time() - lgb_time))
print("[{}] Prediction completed.".format(time.time() - lgb_time))
dt = DecisionTreeRegressor(random_state=42, max_depth=2 )<train_model>
final_time = time.time()
final_ridge_pred1 = clf1.predict(test_X)
 print("[{}] First Ridge prediction completed.".format(time.time() - final_time))
final_ridge_pred2 = clf2.predict(test_X)
 print("[{}] Second Ridge prediction completed.".format(time.time() - final_time))
final_lgb_pred = lgb_model.predict(test_X)
 print("[{}] Lgb prediction completed.".format(time.time() - final_time))
final_pred = 0.25*final_ridge_pred1 + 0.25*final_ridge_pred2 + 0.5*final_lgb_pred
submission = pd.DataFrame(data=df_test[['test_id']])
 submission['price'] = np.expm1(final_pred)
 submission.to_csv("submission_ridge_lgbm.csv", index=False )
<import_modules><EOS>