Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<find_best_params>
lemma = WordNetLemmatizer()
learner.lr_find(start_lr=1e-6 )<define_search_space>
def clean(text):
text = re.sub('\s+', ' ', text)
 text = re.sub("'", "", text)
 return text
def doc_to_words(doc, lemma):
words = [w for w in gensim.utils.simple_preprocess(str(doc), deacc=True)]
words = [lemma.lemmatize(w)for w in words]
return words
max_lr_choice = 5e-4<train_model>
def get_dict_with_split(text_arr):
clean_text_arr = [clean(text)for text in text_arr]
words = [doc_to_words(text, lemma)for text in clean_text_arr]
words_dict = [{w: 1 for w in ww} for ww in words]
return words_dict
learner.fit_one_cycle(4, max_lr=max_lr_choice )<save_model>
def get_dict_without_split(text_arr):
words_dict = [{w: 1} for w in text_arr]
return words_dict
learner.save('imgsize224-stage2' )<load_pretrained>
def df_drop_price(df):
try:
df = df.drop(['price'], axis=1)
 except KeyError:
pass
return df
np.random.seed(123)
 data = ImageDataBunch.from_df(
 path='/kaggle/working/train',
df=cleaned,
valid_pct=0.2,
ds_tfms=get_transforms(flip_vert=False),
size=300,
bs=32
).normalize(imagenet_stats )<choose_model_class>
def df_fillna(df):
df['category_name'] = df['category_name'].fillna('NAN')
 df['brand_name'] = df['brand_name'].fillna('NAN')
 df['item_description'] = df['item_description'].fillna('NAN')
 return df
%%capture
learner = cnn_learner(data, models.resnet50, metrics=error_rate ).to_fp16()
learner.load('imgsize224-stage2' )<find_best_params>
def df_append_cats(df):
df['category_level1'] = df['category_name']
df['category_level2'] = df['category_name']
df['category_level3'] = df['category_name']
return df
learner.lr_find(start_lr=1e-6, end_lr=1e-2, stop_div=False )<define_search_space>
def split_cats(arr):
for i in range(len(arr)) :
try:
s = arr[i,3].split('/')
 arr[i,7] = s[0]
arr[i,8] = s[1]
arr[i,9] = s[2]
except Exception:
arr[i,7] = 'NAN'
arr[i,8] = 'NAN'
arr[i,9] = 'NAN'
return arr
max_lr_choice = 5e-5<train_model>
def fill_na_descriptions(arr):
arr[np.where(arr[:,6]=='No description yet')[0],6] = arr[np.where(arr[:,6]=='No description yet')[0],1]
arr[np.where(arr[:,6]=='NAN')[0],6] = arr[np.where(arr[:,6]=='NAN')[0],1]
return arr
learner.fit_one_cycle(4, max_lr=max_lr_choice )<save_model>
def get_X_from_df(df):
df = df_drop_price(df)
 df = df_fillna(df)
 df = df_append_cats(df)
 X = np.array(df)
 X = split_cats(X)
 X = fill_na_descriptions(X)
 return X
learner.save('imgsize300-stage1' )<find_best_params>
INPUT_PATH = r'.. /input'
learner.unfreeze()
learner.lr_find(start_lr=1e-6, end_lr=1e-2, stop_div=False )<train_model>
df_train = pd.read_table(os.path.join(INPUT_PATH, 'train.tsv'), engine='c' )
learner.fit_one_cycle(4, max_lr=slice(1e-5, 1e-4))<save_model>
logprice = np.log1p(df_train['price'] )
learner.save('imgsize300-stage2' )<categorify>
X_train = get_X_from_df(df_train )
learner = learner.to_fp32()
learner.export()<define_variables>
print('names')
 words_dict = get_dict_with_split(X_train[:,1])
 print('dv_names')
 dv_names = DictVectorizer(sparse=True)
 dv_names.fit(words_dict)
 names_vec_matrix = dv_names.transform(words_dict )
test_fnames = get_image_files('/kaggle/working/test')
 len(test_fnames )<define_variables>
dv_names = 0
ids = [int(re.findall(r'\d+', str(fname)) [0])for fname in test_fnames]
ids[:5]<define_variables>
print('category1')
 words_dict = get_dict_with_split(X_train[:,7])
 print('dv_cat1')
 dv_cat1 = DictVectorizer(sparse=True)
 dv_cat1.fit(words_dict)
 cat1d_vec_matrix = dv_cat1.transform(words_dict )
learner.path<load_pretrained>
dv_cat1 = 0
learner = load_learner(path=learner.path, test=test_fnames )<predict_on_test>
print('category2')
 words_dict = get_dict_with_split(X_train[:,8])
 print('dv_cat2')
 dv_cat2 = DictVectorizer(sparse=True)
 dv_cat2.fit(words_dict)
 cat2d_vec_matrix = dv_cat2.transform(words_dict )
preds, labels = learner.get_preds(ds_type=DatasetType.Test)
 preds[:5]<prepare_output>
dv_cat2 = 0
d = {'id': ids, 'label': preds[:, 1]}
submission = pd.DataFrame(data=d)
 submission = submission.sort_values(by='id')
 submission.head()<save_to_csv>
print('category3')
 words_dict = get_dict_with_split(X_train[:,9])
 print('dv_cat3')
 dv_cat3 = DictVectorizer(sparse=True)
 dv_cat3.fit(words_dict)
 cat3d_vec_matrix = dv_cat3.transform(words_dict )
submission.to_csv('submission.csv', index=False )<save_to_csv>
dv_cat3 = 0
submission.to_csv('submission.csv', index=False )<import_modules>
print('description')
 words_dict = get_dict_with_split(X_train[:,6])
 print('dv_des')
 dv_des = DictVectorizer(sparse=True)
 dv_des.fit(words_dict)
 des_vec_matrix = dv_des.transform(words_dict )
import pandas as pd
import numpy as np
import scipy<import_modules>
dv_des = 0
from sklearn.linear_model import Ridge,LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer<load_from_csv>
condition = X_train[:,2]
shipping = X_train[:,5]
print('MinMaxScaler')
 mms = MinMaxScaler()
mms.fit(condition.reshape(-1,1))
condition_arr_n = mms.transform(condition.reshape(-1,1))
shipping_arr_n = shipping.reshape(-1,1 ).astype('float64' )
train = pd.read_csv('.. /input/train.tsv', sep = '\t')
 test = pd.read_csv('.. /input/test.tsv', sep = '\t')
<define_variables>
mms = 0
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000<count_values>
print('brands')
 words_dict = get_dict_without_split(X_train[:,4])
 print('dv_brand')
 dv_brand = DictVectorizer(sparse=True)
 dv_brand.fit(words_dict)
 brand_vec_matrix = dv_brand.transform(words_dict )
train.brand_name.value_counts()<concatenate>
dv_brand = 0
dt = pd.concat([train,test],0)
<count_values>
print('X_modified')
 X_modified = hstack(( names_vec_matrix,cat1d_vec_matrix,cat2d_vec_matrix,cat3d_vec_matrix,shipping_arr_n,condition_arr_n,brand_vec_matrix,des_vec_matrix))
dt.brand_name.value_counts()<define_variables>
names_vec_matrix = 0
cat1d_vec_matrix = 0
cat2d_vec_matrix = 0
cat3d_vec_matrix = 0
des_vec_matrix = 0
shipping_arr_n = 0
condition_arr_n = 0
words_dict = 0
nrow_train = train.shape[0] 
<prepare_x_and_y>
t1 = time.time()
print('LGBMRegressor')
 lgb_r = lgb.LGBMRegressor(subsample_for_bin = 100000, reg_lambda = 0.0, reg_alpha = 0.0, num_leaves = 51,
n_estimators = 1000, min_split_gain = 0.0, min_child_samples = 20, max_depth = -1,
learning_rate = 0.1, importance_type = 'split', class_weight = None,
boosting_type = 'gbdt')
 lgb_r.fit(X_modified, logprice)
 t2 = time.time()
print(t2-t1 )
target = np.log1p(train.price)
 target<data_type_conversions>
lgb_r = 0
dt.category_name = dt.category_name.fillna('Other' ).astype('category')
 dt.brand_name = dt.brand_name.fillna('unknown' )<count_missing_values>
df_test = pd.read_table(os.path.join(INPUT_PATH, 'test_stg2.tsv'), engine='c' )
dt.isnull().sum()<count_values>
X_test = get_X_from_df(df_test )
pop_brands = dt.brand_name.value_counts().index[:NUM_BRANDS]<feature_engineering>
dv_names = pickle.load(open('.. /dv_names.dv', 'rb'))
dt.loc[~dt.brand_name.isin(pop_brands), 'brand_name'] = 'Other'<data_type_conversions>
print('names')
 words_dict = get_dict_with_split(X_test[:,1])
 print('dv_names')
 names_vec_matrix = dv_names.transform(words_dict )
dt.item_description = dt.item_description.fillna('None')
 dt.item_condition_id = dt.item_condition_id.astype('category')
 dt.brand_name = dt.brand_name.astype('category')
<feature_engineering>
dv_names = 0
count = CountVectorizer(min_df = NAME_MIN_DF)
 x_name = count.fit_transform(dt.name )<import_modules>
dv_cat1 = pickle.load(open('.. /dv_cat1.dv', 'rb'))
from scipy.sparse import find<feature_engineering>
print('category1')
 words_dict = get_dict_with_split(X_test[:,7])
 print('dv_cat1')
 cat1d_vec_matrix = dv_cat1.transform(words_dict )
count_category = CountVectorizer()<feature_engineering>
dv_cat1 = 0
x_category = count_category.fit_transform(dt.category_name )<feature_engineering>
dv_cat2 = pickle.load(open('.. /dv_cat2.dv', 'rb'))
count_desc = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = 'english')
<categorify>
print('category2')
 words_dict = get_dict_with_split(X_test[:,8])
 print('dv_cat2')
 cat2d_vec_matrix = dv_cat2.transform(words_dict )
vect_brand = LabelBinarizer(sparse_output = True)
 x_brand = vect_brand.fit_transform(dt.brand_name )<normalization>
dv_cat2 = 0
x_dummies = scipy.sparse.csr_matrix(pd.get_dummies(dt[['item_condition_id','shipping']],sparse = True ).values )<concatenate>
dv_cat3 = pickle.load(open('.. /dv_cat3.dv', 'rb'))
X = scipy.sparse.hstack(( x_dummies,
x_descp,
x_brand,
x_category,
x_name)).tocsr()<prepare_x_and_y>
print('category3')
 words_dict = get_dict_with_split(X_test[:,9])
 print('dv_cat3')
 cat3d_vec_matrix = dv_cat3.transform(words_dict )
x_train = X[:nrow_train]<choose_model_class>
dv_cat3 = 0
clf = Ridge(solver = 'lsqr',fit_intercept = False )<prepare_x_and_y>
dv_des = pickle.load(open('.. /dv_des.dv', 'rb'))
y_train = target<train_model>
print('description')
 words_dict = get_dict_with_split(X_test[:,6])
 print('dv_des')
 des_vec_matrix = dv_des.transform(words_dict )
clf.fit(x_train,y_train )<split>
dv_des = 0
x_test = X[nrow_train:]<predict_on_test>
mms = pickle.load(open('.. /mms.mms', 'rb'))
preds = clf.predict(x_test )<save_to_csv>
condition = X_test[:,2]
shipping = X_test[:,5]
print('MinMaxScaler')
 condition_arr_n = mms.transform(condition.reshape(-1,1))
shipping_arr_n = shipping.reshape(-1,1 ).astype('float64' )
test["price"] = np.expm1(preds)
 test[['test_id','price']].to_csv('ridge_clf.csv',index = False )<compute_test_metric>
mms = 0
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5 
<load_from_csv>
dv_brand = pickle.load(open('.. /dv_brand.dv', 'rb'))
print("Loading data...")
 train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(train.shape)
 print(test.shape )<categorify>
print('brands')
 words_dict = get_dict_without_split(X_test[:,4])
 print('dv_brand')
 brand_vec_matrix = dv_brand.transform(words_dict )
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
print('X_test_modified')
 X_test_modified = hstack(( names_vec_matrix,cat1d_vec_matrix,cat2d_vec_matrix,cat3d_vec_matrix,shipping_arr_n,condition_arr_n,brand_vec_matrix,des_vec_matrix))
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3)
<feature_engineering>
names_vec_matrix = 0
cat1d_vec_matrix = 0
cat2d_vec_matrix = 0
cat3d_vec_matrix = 0
des_vec_matrix = 0
shipping_arr_n = 0
condition_arr_n = 0
words_dict = 0
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<define_variables>
lgb_r = pickle.load(open('.. /lgb_r.lgb', 'rb'))
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<normalization>
print('y_pred')
 y_pred = lgb_r.predict(X_test_modified )
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()<split>
print('y_pred_e')
 y_pred_e = np.array([np.power(np.e,y)-1 for y in y_pred] )
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )<categorify>
for i in np.where(y_pred_e < 0)[0]:
y_pred_e[i] = np.mean(price )
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
submission = df_test[['test_id']]
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.1
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="sigmoid" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary() 
<train_model>
submission['price'] = y_pred_e
<predict_on_test><EOS> 
submission.to_csv('lgbm_submission.csv', index = False )