Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<predict_on_test>
%%time
time_start=datetime.datetime.now()
session_conf = tf.ConfigProto(intra_op_parallelism_threads=10, inter_op_parallelism_threads=1)
 backend.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))
os.environ['PYTHONHASHSEED'] = '123'
np.random.seed(123)
 np.random.seed(123)
 random.seed(123)
 set_random_seed(123 )
test_pred_lr=lr.predict_proba(test_oh )<prepare_output>
%%time
np.random.seed(123)
 train = pd.read_csv('.. /input/train.tsv',sep='\t')
 test = pd.read_csv('.. /input/test.tsv',sep='\t')
 train=train[(train.price>=3)]
full_df=pd.concat([train,test],ignore_index=True,axis=0 )
output_dict = {'id': test.id,
'target': test_pred_lr[:,1]}
output = pd.DataFrame(output_dict, columns = ['id', 'target'])
 output.head(10 )<save_to_csv>
%%time
all_brands = set(full_df['brand_name'].values)
 full_df.brand_name.fillna('missing',inplace=True)
 premissing = len(full_df.loc[full_df.brand_name == 'missing'])
 def brandfinder(line):
brand = line[0]
name = line[1]
namesplit = name.split(' ')
 if brand == 'missing':
for x in namesplit:
if x in all_brands:
return name
if name in all_brands:
return name
return brand
full_df['brand_name'] = full_df[['brand_name','name']].apply(brandfinder, axis = 1)
 found = premissing-len(full_df.loc[full_df['brand_name'] == 'missing'])
 print(found )
output.to_csv('submission_One_Hot.csv', index=False )<load_from_csv>
%%time
def preprocess_data(df):
df.item_description.replace('No description yet',value='missing',inplace=True)
 df.item_description.fillna('missing',inplace=True)
 df.category_name.fillna('missing/missing/missing',inplace=True)
 df['main_category']=df.category_name.apply(lambda x:x.split('/')[0])
 df['sub_category1']=df.category_name.apply(lambda x:x.split('/')[1])
 df['sub_category2']=df.category_name.apply(lambda x:x.split('/')[2])
 preprocess_data(full_df )
%%time
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<drop_column>
%%time
print("Handling categorical variables")
 le=LabelEncoder()
full_df['category_name']=le.fit_transform(full_df.category_name)
 full_df['brand_name'] = le.fit_transform(full_df.brand_name)
 full_df['main_category']=le.fit_transform(full_df.main_category)
 full_df['sub_category1']=le.fit_transform(full_df.sub_category1)
 full_df['sub_category2']=le.fit_transform(full_df.sub_category2)
 del le
%%time
target = train['target'].values
train_id = train['id'].tolist()
test_id = test['id'].tolist()
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<count_duplicates>
%%time
train=full_df[:n_train]
test=full_df[n_train:]
dtrain,dvalid=train_test_split(train,random_state=123,test_size=0.01)
def reduce_dim(df, column):
for index, dup in df[column].duplicated(keep=False ).iteritems() :
if dup == False:
df.at[index, column] = -1
new_index = {idx:i for i, idx in enumerate(df[column].unique())}
df[column] = df[column].map(new_index)
 return df
def data_treatment(df):
for col in list(df.columns):
if col.startswith('bin'):
bins = df[col].unique()
df[col] = df[col].map({bins[0]:0, bins[1]:1} ).astype('int8')
 df['ord_5'] = df['ord_5'].str[0]
df['isweekend'] =(df['day'] >= 5 ).astype('int8')
 return df<concatenate>
%%time
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
if NUM_WORDS is None:
MAX_TEXT=len(word_index)+1
else:
MAX_TEXT=NUM_WORDS+1
MAX_CATEGORY_MAIN = np.max([full_df.main_category.max() ])+1
MAX_CATEGORY_SUB1 = np.max([full_df.sub_category1.max() ])+1
MAX_CATEGORY_SUB2 = np.max([full_df.sub_category2.max() ])+1
MAX_BRAND = np.max([full_df.brand_name.max() ])+1
MAX_CONDITION = np.max([full_df.item_condition_id.max() ])+1
MAX_DESC_LEN = np.max([full_df.desc_len.max() ])+1
MAX_NAME_LEN = np.max([full_df.name_len.max() ])+1
%%time
whole = pd.concat([train, test])
 whole = data_treatment(whole)
 cat_cols = whole.columns[5:-1]
non_cat_cols = list(set(whole.columns)-set(cat_cols))
for category in cat_cols:
whole = reduce_dim(whole, category )<categorify>
%%time
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_desc, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset[["brand_name"]]),
'item_condition': to_categorical(dataset[["item_condition_id"]]),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'main_category':np.array(dataset[["main_category"]]),
'sub_category1': np.array(dataset[["sub_category1"]]),
'sub_category2':np.array(dataset[["sub_category2"]]),
'category_name':np.array(dataset[["category_name"]]),
'item_condition2':np.array(dataset[["item_condition_id"]]),
}
return X
X_train = get_keras_data(dtrain)
 y_train=np.log1p(dtrain.price)
 X_valid = get_keras_data(dvalid)
 y_valid = np.log1p(dvalid.price)
 X_test = get_keras_data(test)
 test_id=test['test_id'].astype('int32')
 del dtrain
del dvalid
del test
del train
del full_df
%%time
enc = OneHotEncoder(handle_unknown='ignore')
 enc = enc.fit(whole[cat_cols])
 whole_ohe = enc.transform(whole[cat_cols])
 whole_ohe = hstack(( whole_ohe, whole[non_cat_cols]))
whole_ohe = whole_ohe.tocsr()
train_ohe = whole_ohe[:train.shape[0], :]
test_ohe = whole_ohe[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )<train_on_grid>
%%time
def get_cnn_model(lr=0.001,decay=0):
dr_r=0
name=Input(shape=[X_train['name'].shape[1]],name='name')
 category_name=Input(shape=[1],name='category_name')
 desc=Input(shape=[X_train['item_desc'].shape[1]],name='item_desc')
 brand=Input(shape=[1],name='brand_name')
 item_condition=Input(shape=[X_train['item_condition'].shape[1]],name='item_condition')
 shipping=Input(shape=[X_train['num_vars'].shape[1]],name='num_vars')
 desc_len=Input(shape=[1],name='desc_len')
 name_len=Input(shape=[1],name='name_len')
 main_category=Input(shape=[1],name='main_category')
 sub_category1=Input(shape=[1],name='sub_category1')
 sub_category2=Input(shape=[1],name='sub_category2')
 emb_name=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(name)
 emb_desc=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(desc)
 emb_brand=Embedding(MAX_BRAND,10,embeddings_initializer='glorot_uniform' )(brand)
 emb_desc_len=Embedding(MAX_DESC_LEN,5,embeddings_initializer='glorot_uniform' )(desc_len)
 emb_name_len=Embedding(MAX_NAME_LEN,5,embeddings_initializer='glorot_uniform' )(name_len)
 emb_main_category=Embedding(MAX_CATEGORY_MAIN,10,embeddings_initializer='glorot_uniform' )(main_category)
 emb_sub_category1=Embedding(MAX_CATEGORY_SUB1,10,embeddings_initializer='glorot_uniform' )(sub_category1)
 emb_sub_category2=Embedding(MAX_CATEGORY_SUB2,10,embeddings_initializer='glorot_uniform' )(sub_category2)
 emb_item_condition =Embedding(MAX_CONDITION, 5,embeddings_initializer='glorot_uniform' )(item_condition)
 conv_name=Conv1D(filters=32,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_name)
 max_pooling_name=GlobalMaxPooling1D()(conv_name)
 conv_desc=Conv1D(filters=64,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_desc)
 max_pooling_desc=GlobalMaxPooling1D()(conv_desc)
 main_l=concatenate([
max_pooling_name,
max_pooling_desc,
Flatten()(emb_main_category),
Flatten()(emb_sub_category1),
Flatten()(emb_sub_category2),
Flatten()(emb_name_len),
Flatten()(emb_desc_len),
Flatten()(emb_brand),
item_condition,
shipping,
])
 main_l=Dropout(dr_r )(Dense(256,activation='relu',kernel_initializer='he_uniform' )(main_l))
main_l=Dropout(dr_r )(Dense(128,activation='relu',kernel_initializer='he_uniform' )(main_l))
output=Dense(1,kernel_initializer='he_uniform' )(main_l)
 cnn_model=Model(inputs=[name,desc,brand,item_condition,shipping,desc_len,name_len,category_name,main_category,sub_category1,sub_category2],outputs=output)
 optimizer=Adam(lr=lr,decay=decay)
 cnn_model.compile(loss="mse", optimizer=optimizer)
 return cnn_model
def new_rnn_model(lr=0.001, decay=0.0):
dr_r=0
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 item_condition = Input(shape=[1], name="item_condition2")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 desc_len = Input(shape=[1], name="desc_len")
 name_len = Input(shape=[1], name="name_len")
 subcat_0 = Input(shape=[1], name="main_category")
 subcat_1 = Input(shape=[1], name="sub_category1")
 subcat_2 = Input(shape=[1], name="sub_category2")
 brand_enc=Input(shape=[1],name='brand_name_enc')
 item_condition_enc=Input(shape=[1],name='item_condition_enc')
 shipping_enc=Input(shape=[1],name='shipping_enc')
 main_category_enc=Input(shape=[1],name='main_category_enc')
 sub_category1_enc=Input(shape=[1],name='sub_category1_enc')
 sub_category2_enc=Input(shape=[1],name='sub_category2_enc')
 emb_name = Embedding(MAX_TEXT, 50,name='emb_name',embeddings_initializer='glorot_uniform' )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50,name='emb_item_desc',embeddings_initializer='glorot_uniform' )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10,name='emb_brand_name',embeddings_initializer='glorot_uniform' )(brand_name)
 emb_item_condition = Embedding(MAX_CONDITION,5,name='emb_item_condition',embeddings_initializer='glorot_uniform' )(item_condition)
 emb_desc_len = Embedding(MAX_DESC_LEN, 5,name='emb_desc_len',embeddings_initializer='glorot_uniform' )(desc_len)
 emb_name_len = Embedding(MAX_NAME_LEN, 5,name='emb_name_len',embeddings_initializer='glorot_uniform' )(name_len)
 emb_subcat_0 = Embedding(MAX_CATEGORY_MAIN, 10,name='emb_sub_cat_0',embeddings_initializer='glorot_uniform' )(subcat_0)
 emb_subcat_1 = Embedding(MAX_CATEGORY_SUB1, 10,name='emb_sub_cat_1',embeddings_initializer='glorot_uniform' )(subcat_1)
 emb_subcat_2 = Embedding(MAX_CATEGORY_SUB2, 10,name='emb_sub_cat_2',embeddings_initializer='glorot_uniform' )(subcat_2)
 rnn_layer1 = GRU(16,kernel_initializer='glorot_uniform' )(emb_item_desc)
 rnn_layer2 = GRU(8,kernel_initializer='glorot_uniform' )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_item_condition)
 , Flatten()(emb_desc_len)
 , Flatten()(emb_name_len)
 , Flatten()(emb_subcat_0)
 , Flatten()(emb_subcat_1)
 , Flatten()(emb_subcat_2)
 , rnn_layer1
, rnn_layer2
, num_vars,
])
 main_l = Dropout(dr_r )(Dense(512,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(256,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(128,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(64,kernel_initializer='he_uniform',activation='relu' )(main_l))
output = Dense(1,activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name , item_condition,
num_vars, desc_len, name_len, subcat_0, subcat_1,
subcat_2], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss = 'mse', optimizer = optimizer)
 return model
def get_cnn_model2(lr=0.001,decay=0):
dr_r=0
name=Input(shape=[X_train['name'].shape[1]],name='name')
 category_name=Input(shape=[1],name='category_name')
 desc=Input(shape=[X_train['item_desc'].shape[1]],name='item_desc')
 brand=Input(shape=[1],name='brand_name')
 item_condition=Input(shape=[X_train['item_condition'].shape[1]],name='item_condition')
 shipping=Input(shape=[X_train['num_vars'].shape[1]],name='num_vars')
 desc_len=Input(shape=[1],name='desc_len')
 name_len=Input(shape=[1],name='name_len')
 main_category=Input(shape=[1],name='main_category')
 sub_category1=Input(shape=[1],name='sub_category1')
 sub_category2=Input(shape=[1],name='sub_category2')
 emb_name=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(name)
 emb_desc=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(desc)
 emb_brand=Embedding(MAX_BRAND,10,embeddings_initializer='glorot_uniform' )(brand)
 emb_desc_len=BatchNormalization()(desc_len)
 emb_name_len=BatchNormalization()(name_len)
 emb_main_category=Embedding(MAX_CATEGORY_MAIN,10,embeddings_initializer='glorot_uniform' )(main_category)
 emb_sub_category1=Embedding(MAX_CATEGORY_SUB1,10,embeddings_initializer='glorot_uniform' )(sub_category1)
 emb_sub_category2=Embedding(MAX_CATEGORY_SUB2,10,embeddings_initializer='glorot_uniform' )(sub_category2)
 emb_item_condition = Embedding(MAX_CONDITION, 5,embeddings_initializer='glorot_uniform' )(item_condition)
 conv_name=Conv1D(filters=32,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_name)
 max_pooling_name=GlobalMaxPooling1D()(conv_name)
 conv_desc=Conv1D(filters=64,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_desc)
 max_pooling_desc=GlobalMaxPooling1D()(conv_desc)
 main_l=concatenate([
max_pooling_name,
max_pooling_desc,
Flatten()(emb_main_category),
Flatten()(emb_sub_category1),
Flatten()(emb_sub_category2),
emb_name_len,
emb_desc_len,
Flatten()(emb_brand),
item_condition,
shipping
])
 main_l=Dropout(dr_r )(Dense(256,activation='relu',kernel_initializer='he_uniform' )(main_l))
main_l=Dropout(dr_r )(Dense(128,activation='relu',kernel_initializer='he_uniform' )(main_l))
output=Dense(1,kernel_initializer='he_normal' )(main_l)
 cnn_model=Model(inputs=[name,desc,brand,item_condition,shipping,desc_len,name_len,main_category,sub_category1,sub_category2,category_name],outputs=output)
 optimizer=Adam(lr=lr,decay=decay)
 cnn_model.compile(loss="mse", optimizer=optimizer)
 return cnn_model
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, model_type='logreg'):
kf = StratifiedKFold(n_splits=5)
 fold_splits = kf.split(train, target)
 trn_scores = []
cv_scores = []
pred_oof = np.zeros(( train.shape[0]))
pred_full_test = 0
for i,(dev_index, val_index)in enumerate(fold_splits, 1):
print(f'Start fold {i}/5')
 trn_X, val_X = train[dev_index], train[val_index]
trn_y, val_y = target[dev_index], target[val_index]
pred_trn, pred_val, pred_test = model_fn(trn_X, trn_y, val_X, val_y, test, params)
 pred_oof[val_index] = pred_val
pred_full_test += pred_test / 5.0
if eval_fn is not None:
trn_sc = eval_fn(trn_y, pred_trn)
 cv_sc = eval_fn(val_y, pred_val)
 trn_scores.append(trn_sc)
 cv_scores.append(cv_sc)
 print(f'trn score {i}: {trn_sc}')
 print(f'cv score {i}: {cv_sc}')
 print()
print(f'trn scores : {trn_scores}')
 print(f'trn mean score : {np.mean(trn_scores)}')
 print(f'trn std score : {np.std(trn_scores)}')
 print()
print(f'oof cv scores : {eval_fn(target, pred_oof)}')
 print(f'cv scores : {cv_scores}')
 print(f'cv mean score : {np.mean(cv_scores)}')
 print(f'cv std score : {np.std(cv_scores)}')
 print()
results = {'model_type': model_type,
'pred_oof': pred_oof, 'pred_test': pred_full_test,
'trn_scores': trn_scores, 'cv_scores': cv_scores}
return results
def runLR(train_X, train_y, val_X, val_y, test_X, params):
print('Training Logistic Regression...')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predicting 1/3...')
 pred_trn = model.predict_proba(train_X)[:, 1]
print('Predicting 2/3...')
 pred_val = model.predict_proba(val_X)[:, 1]
print('Predicting 3/3...')
 pred_test = model.predict_proba(test_X)[:, 1]
return pred_trn, pred_val, pred_test
lr_params = {'solver': 'lbfgs', 'C': 0.1, 'max_iter': 1000}
results = run_cv_model(train_ohe, test_ohe, target, runLR, lr_params, auc )<save_to_csv>
%%time
BATCH_SIZE = 512*3
epochs = 2
np.random.seed(123)
 exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.005, 0.0001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model= get_cnn_model(lr_init,lr_decay)
 model.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )
submission = pd.DataFrame({'id': test_id, 'target': results['pred_test']})
 submission.to_csv('submission.csv', index=False )<set_options>
%%time
np.random.seed(123)
 BATCH_SIZE = 512*3
epochs = 2
lr_init, lr_fin = 0.004, 0.001
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model2= new_rnn_model(lr_init,lr_decay)
 model2.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )
warnings.filterwarnings("ignore" )<load_from_csv>
%%time
np.random.seed(123)
 BATCH_SIZE = 512*3
epochs = 2
lr_init, lr_fin = 0.005, 0.001
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model3= get_cnn_model2(lr_init,lr_decay)
 model3.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 labels = train.pop('target')
 labels = labels.values
train_id = train.pop("id")
 test_id = test.pop("id" )<categorify>
%%time
cnn_predict=model.predict(X_valid,batch_size=BATCH_SIZE)
 rnn_predict=model2.predict(X_valid,batch_size=BATCH_SIZE)
 cnn_predict2=model3.predict(X_valid,batch_size=BATCH_SIZE)
 X_meta=np.hstack([cnn_predict,rnn_predict,cnn_predict2])
 lm=LinearRegression()
lm.fit(X_meta,y_valid)
 new_pred=lm.predict(X_meta)
 rmsle=np.sqrt(mean_squared_error(y_valid,new_pred))
print('loss:'+str(rmsle))
def ord_to_num(df, col):
keys=np.sort(df[col].unique())
 values=np.arange(len(keys))
map = dict(zip(keys, values))
df[col] = df[col].replace(map)
 ord_to_num(train, 'ord_4')
 ord_to_num(test,'ord_4')
 ord_to_num(train, 'ord_5')
 ord_to_num(test,'ord_5')
 train['ord_4_band'] = pd.qcut(train['ord_4'], 6)
 bands=train.ord_4_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
train['ord_4_band'] = train['ord_4_band'].replace(map_bands)
 test['ord_4_band']=pd.cut(test.ord_4,pd.IntervalIndex(keys_bands))
test['ord_4_band'] = test['ord_4_band'].replace(map_bands)
 train['ord_5_band'] = pd.qcut(train['ord_5'], 6)
 bands=train.ord_5_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
train['ord_5_band'] = train['ord_5_band'].replace(map_bands)
 test['ord_5_band']=pd.cut(test.ord_5,pd.IntervalIndex(keys_bands))
test['ord_5_band'] = test['ord_5_band'].replace(map_bands)
 for col in ["nom_7", "nom_8", "nom_9"]:
train_vals = set(train[col].unique())
 test_vals = set(test[col].unique())
 ex=train_vals ^ test_vals
if ex:
train.loc[train[col].isin(ex), col]="x"
test.loc[test[col].isin(ex), col]="x"<categorify>
%%time
predictions=lm.coef_[0]*model.predict(X_test,batch_size=BATCH_SIZE)+lm.coef_[1]*model2.predict(X_test,batch_size=BATCH_SIZE)+lm.coef_[2]*model3.predict(X_test,batch_size=BATCH_SIZE)+lm.intercept_
predictions=np.exp(predictions)-1
submission=pd.DataFrame(columns=['test_id','price'])
 submission['test_id']=test_id
submission['price']=predictions
submission.to_csv('EnsembleSubmission.csv',index=False )
<split><EOS> 
print('program ended took '+str(datetime.datetime.now() -time_start)+' minutes' )