Kaggle competition: Mercari Price Suggestion Challenge
def cv(df, test, k, feats, y_name):
preds, score, fis = [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i + 1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[: i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[: i*chunk]
rf = RandomForestRegressor(random_state=42, n_jobs=-1, n_estimators=200, min_samples_split=4, max_features=0.9, max_depth=17, oob_score=True)
 rf.fit(train[feats], train[y_name])
 score.append(mean_squared_error(valid[y_name], rf.predict(valid[feats])) **(1/2))
preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return pd.Series(score), pd.Series(preds ).mean() , fis<compute_test_metric>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z
score, preds, fis = cv(df, test, 20, feats, 'rentals' )<feature_engineering>
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip
test['count'] = np.exp(preds )<save_to_csv>
%matplotlib inline
test[['datetime', 'count']].to_csv('rf_cv_wo_day.csv', index=False )<create_dataframe>
train_df = pd.read_csv('train.tsv', sep='\t')
 train_df.shape
fi = pd.Series(pd.DataFrame(fis ).mean().values, index=feats )<define_variables>
test_df = pd.read_csv('test_stg2.tsv', sep='\t')
 test_df.shape
feats = [c for c in feats if c not in ['holiday']]<compute_test_metric>
train_df['price'] = np.log1p(train_df['price'] )
score, preds, fis = cv(df, test, 20, feats, 'rentals' )<feature_engineering>
train_df['shipping'].value_counts()
test['count'] = np.exp(preds )<save_to_csv>
train_df['item_condition_id'].value_counts()
test[['datetime', 'count']].to_csv('rf_cv_wo_holiday.csv', index=False )<import_modules>
train_df['item_description'].value_counts()
import numpy as np
import pandas as pd
import matplotlib as plt
import os 
<count_data_types>
train_df[train_df['item_description'] == 'No description yet']['item_description'].count()
def missingvalues(df):
miss=df.isnull().sum()
misspercent=100*df.isnull().sum() /len(df)
 misvaltable=pd.concat([miss,misspercent],axis=1)
 misvaltable=misvaltable.rename(columns={0:"missing values",1:"missing percent"})
 return misvaltable
df.dtypes.value_counts() 
<count_unique_values>
def split_cat(category_name):
try:
return category_name.split('/')
 except:
return ['Other_Null', 'Other_Null', 'Other_Null']
categoryweather=df.groupby("holiday" ).nunique()
print(categoryweather)
 df1=pd.get_dummies(df['weather'])
 df1.head()<define_variables>
train_df['cat_dae'], train_df['cat_jung'], train_df['cat_so'] = zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 test_df['cat_dae'], test_df['cat_jung'], test_df['cat_so'] = zip(*test_df['category_name'].apply(lambda x: split_cat(x)) )
l=[ 'season', 'holiday', 'workingday', 'weather', 'temp',
'atemp', 'humidity', 'windspeed']
dftarget=df[["casual","registered","count"]].copy()
dftarget.head() 
<drop_column>
print(train_df['cat_dae'].nunique())
 print(train_df['cat_jung'].nunique())
 print(train_df['cat_so'].nunique() )
dfnew=df.copy()
dfnew.drop(["registered","casual","count"],axis=1,inplace=True)
 dfnew.head()<concatenate>
train_df['brand_name'] = train_df['brand_name'].fillna(value='Other_Null')
 train_df['category_name'] = train_df['category_name'].fillna(value='Other_Null')
 train_df['item_description'] = train_df['item_description'].fillna(value='Other_Null')
 test_df['brand_name'] = test_df['brand_name'].fillna(value='Other_Null')
 test_df['category_name'] = test_df['category_name'].fillna(value='Other_Null')
 test_df['item_description'] = test_df['item_description'].fillna(value='Other_Null' )
df2=pd.concat([dfnew,testdf])
 print(df2.shape)
 print(df.shape )<load_from_csv>
train_df.isnull().sum()
df=pd.read_csv(".. /input/train.csv")
 df1=pd.get_dummies(df['weather'])
 df1=df1.rename(columns={1:"clear",2:"misty",3:"snow",4:"heavy snow"})
 df=df.drop(["weather"],axis=1)
 df=pd.concat([df,df1],axis=1)
 df.head()<categorify>
test_df.isnull().sum()
df1=pd.get_dummies(df2["weather"])
 df2.drop(["weather"],axis=1,inplace=True)
 df2=pd.concat([df2,df1],axis=1)
 df2.head() 
<rename_columns>
train_df['brand_name'].nunique()
df2=df2.rename(columns={1:"clear",2:"misty",3:"snow",4:"heavy snow"})
 df2.head()<categorify>
train_df['brand_name'].value_counts() [:5]
df1=pd.get_dummies(df2["season"])
 df3=pd.concat([df2,df1],axis=1)
 df3=df3.rename(columns={1:"spring",2:"summer",3:"fall",4:"winter"})
 df3.head()<categorify>
train_df['name'].nunique()
df.shape
df0=pd.get_dummies(df["season"])
 df4=pd.concat([df,df0],axis=1)
 df4=df4.rename(columns={1:"spring",2:"summer",3:"fall",4:"winter"})
 df4.head()<feature_engineering>
train_df['name'].value_counts() [:10]
df4["weekend"]=[abs(1-abs(x-y)) for x,y in zip(df4["workingday"],df4["holiday"])]
df4.head()<count_unique_values>
cnt_vec = CountVectorizer()
X_train_name = cnt_vec.fit_transform(train_df['name'])
 X_test_name = cnt_vec.transform(test_df['name'] )
print(df4.groupby("weekend")["datetime"].nunique())
 print(df4.groupby("holiday")["datetime"].nunique())
 print(df4.groupby("workingday")["datetime"].nunique() )<feature_engineering>
tfidf_descp = TfidfVectorizer(max_features=50000, ngram_range=(1, 3), stop_words='english')
 X_train_descp = tfidf_descp.fit_transform(train_df['item_description'])
 X_test_descp = tfidf_descp.transform(test_df['item_description'] )
df3["weekend"]=[abs(1-abs(x-y)) for x,y in zip(df3["workingday"],df3["holiday"])]<data_type_conversions>
lb_brand_name = LabelBinarizer(sparse_output=True)
 X_train_brand = lb_brand_name.fit_transform(train_df['brand_name'])
 X_test_brand = lb_brand_name.transform(test_df['brand_name'])
 lb_item_cond_id = LabelBinarizer(sparse_output=True)
 X_train_item_condition_id = lb_item_cond_id.fit_transform(train_df['item_condition_id'])
 X_test_item_condition_id = lb_item_cond_id.transform(test_df['item_condition_id'])
 lb_shipping = LabelBinarizer(sparse_output=True)
 X_train_shipping = lb_shipping.fit_transform(train_df['shipping'])
 X_test_shipping = lb_shipping.transform(test_df['shipping'])
 lb_cat_dae = LabelBinarizer(sparse_output=True)
 X_train_cat_dae = lb_cat_dae.fit_transform(train_df['cat_dae'])
 X_test_cat_dae = lb_cat_dae.transform(test_df['cat_dae'])
 lb_cat_jung = LabelBinarizer(sparse_output=True)
 X_train_cat_jung = lb_cat_jung.fit_transform(train_df['cat_jung'])
 X_test_cat_jung = lb_cat_jung.transform(test_df['cat_jung'])
 lb_cat_so = LabelBinarizer(sparse_output=True)
 X_train_cat_so = lb_cat_so.fit_transform(train_df['cat_so'])
 X_test_cat_so = lb_cat_so.transform(test_df['cat_so'] )
df4["time"]=pd.to_datetime(df4["datetime"])
 df4.head()<feature_engineering>
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 X_train = hstack(sparse_matrix_list ).tocsr()
print(type(X_train), X_train.shape)
 del X_train
gc.collect()
df4["hours"]=df4["time"].dt.hour
df4.head()<data_type_conversions>
def rmsle(y, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y_pred), 2)))
 def evaluate_orig_price(y_test, preds):
preds_exmpm = np.expm1(preds)
 y_test_exmpm = np.expm1(y_test)
 return rmsle(y_test_exmpm, preds_exmpm )
df3["time"]=pd.to_datetime(df3["datetime"])
 df3.head()<feature_engineering>
def model_train_predict(model, matrix_list):
X = hstack(matrix_list ).tocsr()
X_train, X_test, y_train, y_test = train_test_split(X, train_df['price'], test_size=0.2)
 model.fit(X_train, y_train)
 preds = model.predict(X_test)
 del X, X_train, X_test, y_train
gc.collect()
return preds, y_test
df3["hours"]=df3["time"].dt.hour
df3.head()<drop_column>
linear_model = Ridge(solver='lsqr', fit_intercept=False)
 sparse_matrix_list =(X_train_name, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)
 print('Item Description을 제외했을 때 rmsle 값:', evaluate_orig_price(y_test, linear_preds))
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)
 print('Item Description을 포함한 rmsle 값:', evaluate_orig_price(y_test, linear_preds))
df4.drop(["time"],axis=1,inplace=True)
 df3.drop(["time"],axis=1,inplace=True )<feature_engineering>
lgbm_model = LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125)
 sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 lgbm_preds, y_test = model_train_predict(model=lgbm_model, matrix_list=sparse_matrix_list)
 print('LightGBM rmsle 값:', evaluate_orig_price(y_test, lgbm_preds))
df4["logcasual"]=np.log(df4["casual"]+1)
 df4["logcasual"]=np.log(df4["casual"]+1)
 df4["logcasual"]=np.log(df4["casual"]+1)
 df4.head()<feature_engineering>
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 X_train = hstack(sparse_matrix_list ).tocsr()
X_train
df4["logregistered"]=np.log(df4["registered"]+1)
 df4["logcount"]=np.log(df4["count"]+1)
 df4.head()<data_type_conversions>
y_train = train_df['price']
y_train
df4["time"]=pd.to_datetime(df4["datetime"])
 df4.head()<feature_engineering>
lgbm_model = LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125)
 lgbm_model.fit(X_train, y_train )
df4["day"]=df4["time"].dt.day
df4.head(25 )<feature_engineering>
sparse_matrix_list =(X_test_name, X_test_descp, X_test_brand, X_test_item_condition_id, X_test_shipping, X_test_cat_dae, X_test_cat_jung, X_test_cat_so)
 X_test = hstack(sparse_matrix_list ).tocsr()
df4["day"]=df4["time"].dt.dayofweek
df4.head()<feature_engineering>
preds = lgbm_model.predict(X_test)
 preds
df4["day"]=df4["time"].dt.dayofweek
df4.head(25 )<drop_column>
preds = np.expm1(preds)
 preds
df5=df.copy()
df5.drop(["holiday","workingday","season"],axis=1,inplace=True)
 df5.corr()<feature_engineering>
submission = pd.read_csv('sample_submission_stg2.csv')
 submission
df4["year"]=df4['time'].dt.year
df4.head()<count_unique_values>
submission.loc[:, 'price'] = preds
submission
df4.groupby("weekend")["datetime"].nunique()<feature_engineering>
submission.to_csv('submission.csv', index=False )
df4["month"]=df4["time"].dt.month
df4.head()<categorify>
submission.to_csv('submission.csv', index=False )