Kaggle competition: Mercari Price Suggestion Challenge
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3,1]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = Dropout (.5 )(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu" )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 nums = Dropout(0.5 )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.2 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss='mse',)
 return model<prepare_x_and_y>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5
nfolds=5
folds = StratifiedKFold(n_splits=nfolds,shuffle=True, random_state=15)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
x_test = get_keras_data(test_df, desc_embs[len(train_df):])
 adoptions_keras = np.zeros(( len(test_df),))
oof_train_keras = np.zeros(( train_df.shape[0]))
i =0
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"]):
x_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 x_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(x_valid, y_valid), interval=1)
 history = model.fit(x_train, y_train, batch_size=batch_size, validation_data=(x_valid, y_valid),
epochs=20, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(x_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(x_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)
 avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)
 coeffs = eval_predict(y_valid, model.predict(x_valid, batch_size=1000), ret_coeffs=True)
 adoptions_keras += model.predict(x_test, batch_size=batch_size ).reshape(-1,)
 oof_train_keras[valid_idx] = y_valid_pred
print("
avg train kappa:", avg_train_kappa/nfolds,)
 print("
avg valid kappa:", avg_valid_kappa/nfolds, )<count_values>
print("Loading data...")
 train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(train.shape)
 print(test.shape )
Counter(X_train['AdoptionSpeed'])
<count_values>
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )
Counter(X_train['AdoptionSpeed'])
<predict_on_test>
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3)
coeffs[0] = 1.645
coeffs[1] = 2.115
coeffs[3] = 2.84
print('True Counter',Counter(X_train['AdoptionSpeed']))
train_pred_keras = eval_predict(y_pred=oof_train_keras, coeffs=list(coeffs)).astype(int)
 print('Train pred Counter',Counter(train_pred_keras))
test_pred_keras = eval_predict(y_pred=adoptions_keras/nfolds, coeffs=list(coeffs)).astype(int)
 print('Test pred Counter',Counter(test_pred_keras))<concatenate>
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )
train_stack = np.vstack([train_pred_keras, train_predictions_lgb] ).transpose()
test_stack = np.vstack([test_pred_keras, test_predictions_lgb] ).transpose()<import_modules>
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1
from sklearn.linear_model import Ridge 
<train_model>
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(train_stack.shape[0])
 predictions = np.zeros(test_stack.shape[0])
 qwk_scores = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, X_train['AdoptionSpeed'].values)) :
trn_data, trn_y = train_stack[trn_idx], X_train['AdoptionSpeed'].iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], X_train['AdoptionSpeed'].iloc[val_idx].values
clf = Ridge(alpha=1)
 clf.fit(trn_data, trn_y)
 val_pred = clf.predict(val_data)
 pred_val_y_k = eval_predict(y_pred=val_pred, coeffs=list(coeffs)).astype(int)
 print("Valid Counts = ", Counter(val_y))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(val_y, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 oof[val_idx] = clf.predict(val_data)
 predictions += clf.predict(test_stack)/ folds.n_splits
print('{} cv mean QWK score : {}'.format('Stacked_model_ridge', np.mean(qwk_scores)) )<predict_on_test>
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )
pred_final = eval_predict(y_pred=predictions, coeffs=list(coeffs)).astype(int )<count_values>
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )
Counter(pred_final)
<count_values>
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.1
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="sigmoid" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()
Counter(test_pred_keras)
<count_values>
BATCH_SIZE = 100000
epochs = 7
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid, dvalid.target)
 , verbose=1 )
Counter(test_predictions_lgb)
<save_to_csv>
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds
<set_options><EOS> 
submission.to_csv("./myNNsubmission.csv", index=False)
 submission.price.hist()