Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<prepare_x_and_y>
import numpy as np
import pandas as pd
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
parms = {'max_depth':8,
'objective':'reg:logistic',
'eta' :0.3,
'subsample':0.8,
'lambda ' :4,
'colsample_bytree ':0.9,
'colsample_bylevel':1,
'min_child_weight': 10}
modelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,
early_stopping_rounds=30, maximize=False,
verbose_eval=10)
 feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
pred_train['xgb_pred'] = modelx.predict(data_train)
 pred_test['xgb_pred'] = modelx.predict(data_test)
 acc.append(round(r2_score(labels, pred_train['xgb_pred'])* 100, 2))
fig = plt.figure(figsize =(15,15))
axes = fig.add_subplot(111)
 xgb.plot_importance(modelx,ax = axes,height = 0.5)
 plt.show() ;plt.close()<choose_model_class>
train = pd.read_csv('.. /input/train.tsv', sep='\t')
 test = pd.read_csv('.. /input/test.tsv', sep='\t' )
train2 = pd.DataFrame(preprocessing.MinMaxScaler().fit_transform(train),
columns=train.columns, index=train.index)
 logreg = LogisticRegression()
logreg.fit(train2, labels)
 coeff_logreg = pd.DataFrame(train2.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 pred_train['log_pred'] = logreg.predict(train)
 pred_test['log_pred'] = logreg.predict(test)
 acc.append(round(r2_score(labels, pred_train['log_pred'])* 100, 2))
coeff_logreg.sort_values(by='score_logreg', ascending=False )<merge>
%matplotlib inline
coeff_logreg["score_logreg"] = coeff_logreg["score_logreg"].abs()
feature_score = pd.merge(feature_score, coeff_logreg, on='feature' )<merge>
from subprocess import check_output
coeff_linreg["score_linreg"] = coeff_linreg["score_linreg"].abs()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature' )<feature_engineering>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5
feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score = pd.DataFrame(preprocessing.MinMaxScaler().fit_transform(feature_score),
columns=feature_score.columns, index=feature_score.index)
 feature_score['mean'] = feature_score.mean(axis=1)
 pred_train['mean'] = pred_train.mean(axis=1)
 pred_test['mean'] = pred_test.mean(axis=1)
 acc.append(round(r2_score(labels, pred_train['mean'])* 100, 2))
fL = 0.2
flog = 0.5
flin = 0.15
fx = 1-fL-flog-flin
feature_score['total'] = fL*feature_score['score_lgb'] + fx*feature_score['score_xgb'] \
+ flog*feature_score['score_logreg'] + flin*feature_score['score_linreg']
pred_train['total'] = fL*pred_train['lgb_pred'] + fx*pred_train['xgb_pred'] + \
+ flog*pred_train['log_pred'] + flin*pred_train['lin_pred']
pred_test['total'] = fL*pred_test['lgb_pred'] + fx*pred_test['xgb_pred'] + \
+ flog*pred_test['log_pred'] + flin*pred_test['lin_pred']
acc.append(round(r2_score(labels, pred_train['total'])* 100, 2))
feature_score.sort_values('total', ascending=False ).plot(kind='bar', figsize=(20, 10))<sort_values>
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )
feature_score.sort_values('score_logreg', ascending=False )<categorify>
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3 )
columns = [i for i in data.columns]
dummies = pd.get_dummies(data,
columns=columns,
drop_first=True,
sparse=True)
 del data<categorify>
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )
train = dummies.iloc[:train.shape[0], :]
test = dummies.iloc[train.shape[0]:, :]
del dummies<data_type_conversions>
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1
train = train.sparse.to_coo().tocsr()
test = test.sparse.to_coo().tocsr()
train = train.astype("float32")
 test = test.astype("float32" )<train_on_grid>
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()
lr = LogisticRegression(C=0.12,
solver="lbfgs",
tol=0.002,
max_iter=10000)
 lr.fit(train, labels)
 lr_pred = lr.predict_proba(train)[:, 1]
score = roc_auc_score(labels, lr_pred)
 print("score: ", score )<predict_on_test>
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )
submission["id"] = test_id
submission["target"] = lr.predict_proba(test)[:, 1]<save_to_csv>
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )
submission.to_csv("submission.csv", index=False )<save_to_csv>
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.1
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()
submission.to_csv("submission.csv", index=False )<import_modules>
BATCH_SIZE = 20000
epochs = 2
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE, validation_data=(X_valid, dvalid.target)
 , verbose=1 )
import numpy as np
import pandas as pd
import scipy
import sklearn<import_modules>
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder<import_modules>
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds
<load_from_csv><EOS> 
submission.to_csv("sample_submission.csv", index=False)
 submission.price.hist()