Kaggle competition: Mercari Price Suggestion Challenge
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )<load_from_csv>
import time
from scipy.sparse import csr_matrix, hstack
from sklearn.preprocessing import LabelEncoder, Normalizer
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import train_test_split
import sys
import os
import random
import numpy as np
from keras import backend as K
from nltk.corpus import stopwords
import re
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Input, Dropout, Dense, concatenate, GRU, Embedding, Flatten, Activation
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()<data_type_conversions>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )<save_to_csv>
NUM_BRANDS = 4500
NUM_CATEGORIES = 1250
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )<save_to_csv>
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y0), 2)) )
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )<import_modules>
train_df = pd.read_table("train.tsv")
 test_df = pd.read_csv(".. /input/mercari-price-suggestion-challenge/test_stg2.tsv.zip" , sep='\t')
import numpy as np
import pandas as pd
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os<load_from_csv>
def split_cat(text):
try:
return text.split("/")
 except:
return("missing", "missing", "missing" )
df = pd.read_csv('.. /input/train_labels.csv')
 df.head()<define_variables>
def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='No description yet', inplace=True )
SZ = 96
BS = 64
NUM_WORKERS = 4<load_from_csv>
def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category1 = dataset['general_cat'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category2 = dataset['subcat_1'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category3 = dataset['subcat_2'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
dataset.loc[~dataset['general_cat'].isin(pop_category1), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(pop_category2), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(pop_category3), 'subcat_2'] = 'missing'
data =(
 ImageItemList 
.from_csv(path, 'train_labels.csv', folder='train', suffix='.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test(ImageItemList.from_folder('.. /input/test')) 
.transform(get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,max_lighting=0.05, max_warp=0.) , size=SZ, resize_method=ResizeMethod.SQUISH)
.databunch(bs=BS, num_workers=NUM_WORKERS, path = '.')
.normalize(imagenet_stats)
 )<define_variables>
def to_categorical(dataset):
dataset['general_cat'] = dataset['general_cat'].astype('category')
 dataset['subcat_1'] = dataset['subcat_1'].astype('category')
 dataset['subcat_2'] = dataset['subcat_2'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category' )
data.show_batch(3 )<compute_test_metric>
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+' )
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<choose_model_class>
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0
learn = create_cnn(data, densenet161, metrics=[auc_score], ps=0.5 )<train_model>
def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
learn.fit_one_cycle(1, 1e-3 )<train_model>
def normalize_dataset_text(dataset):
dataset['item_description'] = dataset['item_description'].apply(lambda x: normalize_text(x))
dataset['brand_name'] = dataset['brand_name'].apply(lambda x: normalize_text(x))
learn.fit_one_cycle(4, slice(1e-6, 1e-4))<define_variables>
def delete_unseen(dataset):
dataset.loc[~dataset['brand_name'].isin(all_brand), 'brand_name'] = 'missing'
dataset.loc[~dataset['general_cat'].isin(all_general_cat), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(all_subcat_1), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(all_subcat_2), 'subcat_2'] = 'missing'
SZ = 96*2
BS = 64//2
NUM_WORKERS = 0<load_from_csv>
def text_length_feature(dataset, train = True):
if train:
dataset['desc_len'] = dataset['item_description'].apply(lambda x: wordCount(x))
dataset['name_len'] = dataset['name'].apply(lambda x: wordCount(x))
dataset[['desc_len', 'name_len']] = desc_normalizer.fit_transform(dataset[['desc_len', 'name_len']])
 else:
dataset['desc_len'] = dataset['item_description'].apply(lambda x: wordCount(x))
dataset['name_len'] = dataset['name'].apply(lambda x: wordCount(x))
dataset[['desc_len', 'name_len']] = desc_normalizer.transform(dataset[['desc_len', 'name_len']] )
data =(
 ImageItemList 
.from_csv(path, 'train_labels.csv', folder='train', suffix='.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test(ImageItemList.from_folder('.. /input/test')) 
.transform(get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,max_lighting=0.05, max_warp=0.) , size=SZ, resize_method=ResizeMethod.SQUISH)
.databunch(bs=BS, num_workers=NUM_WORKERS, path = '.')
.normalize(imagenet_stats)
 )<prepare_output>
start_time = time.time()
print(strftime("%Y-%m-%d %H:%M:%S", gmtime()))
learn.data = data<train_model>
train_df, dev_df = train_test_split(train_df, random_state=200, train_size=0.70 )
learn.fit_one_cycle(1, 1e-3/2 )<train_model>
train_df = train_df.drop(train_df[(train_df.price < 1.0)].index)
 train_y = np.log1p(train_df["price"])
 dev_y = np.log1p(dev_df["price"] )
learn.fit_one_cycle(1, slice(1e-6/2, 1e-4/2))<predict_on_test>
train_df['general_cat'], train_df['subcat_1'], train_df['subcat_2'] = \
zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 train_df.drop('category_name', axis=1, inplace=True)
 print('[{}] Split categories completed.'.format(time.time() - start_time))
handle_missing_inplace(train_df)
 print('[{}] Handle missing completed.'.format(time.time() - start_time))
cutting(train_df)
 print('[{}] Cut completed.'.format(time.time() - start_time))
to_categorical(train_df)
 print('[{}] Convert categorical completed'.format(time.time() - start_time))
preds = learn.get_preds(ds_type=DatasetType.Test )<load_from_csv>
desc_normalizer = Normalizer()
name_normalizer = Normalizer()
text_length_feature(train_df)
 print('[{}] Calculate length features'.format(time.time() - start_time))
normalize_dataset_text(train_df)
 print('[{}] Normalization text'.format(time.time() - start_time))
df_sub = pd.read_csv('.. /input/sample_submission.csv' ).set_index('id')
 df_sub.head()<data_type_conversions>
all_brand = set(train_df["brand_name"].values)
 all_general_cat = set(train_df["general_cat"].values)
 all_subcat_1 = set(train_df["subcat_1"].values)
 all_subcat_2 = set(train_df["subcat_2"].values)
 le_brand = LabelEncoder()
le_general_cat = LabelEncoder()
le_subcat_1 = LabelEncoder()
le_subcat_2 = LabelEncoder()
le_brand.fit(train_df['brand_name'])
 train_df['encoded_brand_name'] = le_brand.transform(train_df['brand_name'])
 le_general_cat.fit(train_df['general_cat'])
 train_df['encoded_general_cat'] = le_general_cat.transform(train_df['general_cat'])
 le_subcat_1.fit(train_df['subcat_1'])
 train_df['encoded_subcat_1'] = le_subcat_1.transform(train_df['subcat_1'])
 le_subcat_2.fit(train_df['subcat_2'])
 train_df['encoded_subcat_2'] = le_subcat_2.transform(train_df['subcat_2'] )
clean_fname = np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned = clean_fname(data.test_ds.items)
 fname_cleaned = fname_cleaned.astype(str )<feature_engineering>
print("Tokenizing item description")
 tok_desc = Tokenizer()
tok_desc.fit_on_texts(train_df["item_description"].values)
 print("Tokenizing name")
 tok_name = Tokenizer()
tok_name.fit_on_texts(train_df["name"].values)
 print("Transforming text to sequences...")
 train_df['seq_item_description'] = tok_desc.texts_to_sequences(train_df["item_description"].values)
 train_df['seq_name'] = tok_name.texts_to_sequences(train_df["name"].values)
df_sub.loc[fname_cleaned,'label'] = preds[0][:,1].tolist()<save_to_csv>
MAX_NAME_SEQ = 15
MAX_ITEM_DESC_SEQ = 50
MAX_DESC_TEXT = len(tok_desc.word_index)+ 1
MAX_NAME_TEXT = len(tok_name.word_index)+ 1
MAX_BRAND = len(le_brand.classes_)
 MAX_GENCAT = len(le_general_cat.classes_)
 MAX_SUBCAT_1 = len(le_subcat_1.classes_)
 MAX_SUBCAT_2 = len(le_subcat_2.classes_)
 MAX_CONDITION = max(train_df.item_condition_id)+ 1
df_sub.to_csv('sub.csv' )<set_options>
def get_rnn_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset.encoded_brand_name),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'general_cat': np.array(dataset.encoded_general_cat),
'subcat_1': np.array(dataset.encoded_subcat_1),
'subcat_2': np.array(dataset.encoded_subcat_2),
}
return X
%matplotlib inline<set_options>
train_X = get_rnn_data(train_df )
seed = 520
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = True
use_gpu = True<load_from_csv>
np.random.seed(123)
 def rnn_model(lr=0.001, decay=0.0):
name = Input(shape=[MAX_NAME_SEQ], name="name")
 item_desc = Input(shape=[MAX_ITEM_DESC_SEQ], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 general_cat = Input(shape=[1], name="general_cat")
 subcat_1 = Input(shape=[1], name="subcat_1")
 subcat_2 = Input(shape=[1], name="subcat_2")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[1], name="num_vars")
 desc_len = Input(shape=[1], name="desc_len")
 name_len = Input(shape=[1], name="name_len")
 emb_name = Embedding(MAX_NAME_TEXT, 30 )(name)
 emb_item_desc = Dropout(0.05 )(Embedding(MAX_DESC_TEXT, 60 )(item_desc))
emb_brand_name = Embedding(MAX_BRAND, 20 )(brand_name)
 emb_general_cat = Embedding(MAX_GENCAT, 5 )(general_cat)
 emb_subcat_1 = Embedding(MAX_SUBCAT_1, 10 )(subcat_1)
 emb_subcat_2 = Embedding(MAX_SUBCAT_2, 15 )(subcat_2)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_item_condition)
 , Flatten()(emb_general_cat)
 , Flatten()(emb_subcat_1)
 , Flatten()(emb_subcat_2)
 , num_vars
, rnn_layer1
, rnn_layer2
, desc_len
, name_len
])
 main_l = Dropout(0.05 )(Dense(512,kernel_initializer='normal',activation='relu' )(main_l))
main_l = Dropout(0.05 )(Dense(96,kernel_initializer='normal',activation='relu' )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name,
general_cat, subcat_1, subcat_2,
item_condition, num_vars, desc_len, name_len], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss = 'mse', optimizer = optimizer)
 return model
labels = pd.read_csv('.. /input/train_labels.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train_path = '.. /input/train/'
test_path = '.. /input/test/'<count_values>
BATCH_SIZE = 512 * 4
epochs = 3
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(train_X['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.007, 0.002
lr_decay = exp_decay(lr_init, lr_fin, steps)
labels.label.value_counts()<split>
model = rnn_model(lr=lr_init, decay=lr_decay)
 model.fit(train_X, train_y, epochs=epochs, batch_size=BATCH_SIZE, verbose=2 )
train, val = train_test_split(labels, stratify=labels.label, test_size=0.1)
 len(val),len(train )<categorify>
dev_df['general_cat'], dev_df['subcat_1'], dev_df['subcat_2'] = \
zip(*dev_df['category_name'].apply(lambda x: split_cat(x)))
 handle_missing_inplace(dev_df)
 cutting(dev_df)
 text_length_feature(dev_df)
 normalize_dataset_text(dev_df)
 delete_unseen(dev_df)
 to_categorical(dev_df )
class MyDataset(Dataset):
def __init__(self, df_data, data_dir = './', transform=None):
super().__init__()
self.df = df_data.values
self.data_dir = data_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_dir, img_name+'.tif')
 image = mask(cv2.imread(img_path))
if self.transform is not None:
image = self.transform(image)
 return image, label<categorify>
dev_df['encoded_brand_name'] = le_brand.transform(dev_df['brand_name'])
 dev_df['encoded_general_cat'] = le_general_cat.transform(dev_df['general_cat'])
 dev_df['encoded_subcat_1'] = le_subcat_1.transform(dev_df['subcat_1'])
 dev_df['encoded_subcat_2'] = le_subcat_2.transform(dev_df['subcat_2'])
 dev_df['seq_item_description'] = tok_desc.texts_to_sequences(dev_df["item_description"].values)
 dev_df['seq_name'] = tok_name.texts_to_sequences(dev_df["name"].values)
 dev_X = get_rnn_data(dev_df)
 preds_rnn = model.predict(dev_X)
 print("RNN dev RMSLE:", rmsle(np.expm1(dev_y), np.expm1(preds_rnn.flatten())) )
batch_size = 128
trans_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomRotation(20),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0 )<choose_model_class>
test_df['general_cat'], test_df['subcat_1'], test_df['subcat_2'] = \
zip(*test_df['category_name'].apply(lambda x: split_cat(x)))
 test_df.drop('category_name', axis=1, inplace=True)
 handle_missing_inplace(test_df)
 cutting(test_df)
 text_length_feature(test_df)
 normalize_dataset_text(test_df)
 delete_unseen(test_df)
 to_categorical(test_df )
model = models.resnet34(pretrained=True)
<find_best_params>
test_df['encoded_brand_name'] = le_brand.transform(test_df['brand_name'])
 test_df['encoded_general_cat'] = le_general_cat.transform(test_df['general_cat'])
 test_df['encoded_subcat_1'] = le_subcat_1.transform(test_df['subcat_1'])
 test_df['encoded_subcat_2'] = le_subcat_2.transform(test_df['subcat_2'])
 test_df['seq_item_description'] = tok_desc.texts_to_sequences(test_df["item_description"].values)
 test_df['seq_name'] = tok_name.texts_to_sequences(test_df["name"].values)
 test_X = get_rnn_data(test_df)
 preds_rnn_test = model.predict(test_X )
for para in list(model.parameters()):
para.requires_grad=False
for para in list(model.layer3.parameters()):
para.requires_grad=True
for para in list(model.layer4.parameters()):
para.requires_grad=True<choose_model_class>
preds_rnn_test = np.expm1(preds_rnn_test.flatten() )
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(
 nn.Dropout(0.5),
nn.Linear(num_ftrs, 2)
 )<find_best_params>
submission = test_df[["test_id"]]
submission["price"] = preds_rnn_test
params_to_update = []
for name,param in model.named_parameters() :
if param.requires_grad == True:
params_to_update.append(param)
 print("\t",name )<define_variables>
submission.to_csv("submission.csv", index=False )
<choose_model_class><EOS> 
submission.to_csv("submission.csv", index=False )