Kaggle competition: Mercari Price Suggestion Challenge
def create_submission(orig_ids, test_ids, preds):
preds_dict = dict(( k, v)for k, v in zip(test_ids, preds))
pred_cor = [preds_dict[id] for id in orig_ids]
df = pd.DataFrame({'id':orig_ids,'label':pred_cor})
 df.to_csv(f'submission_{auc_val}.csv', header=True, index=False)
 return df<compute_test_metric>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z
test_df = create_submission(orig_ids, test_ids, preds)
 test_df.head()<set_options>
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload<define_variables>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96<load_from_csv>
train = pd.read_table('train.tsv')
 test = pd.read_table('test_stg2.tsv' )
df_trn=pd.read_csv(train_lbl )<feature_engineering>
print("train: ")
 print(pd.isnull(train ).sum())
 print("="*30)
 print("test: ")
 print(pd.isnull(test ).sum() )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)<load_from_csv>
gc.collect()
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )<define_variables>
train['brand_name'] = train['brand_name'].fillna('Nobrand')
 test['brand_name'] = test['brand_name'].fillna('Nobrand' )
data.show_batch(rows=5, figsize=(12,9))<import_modules>
train['category_name'] = train['category_name'].fillna('No/No/No')
 test['category_name'] =test['category_name'].fillna('No/No/No' )
from sklearn.metrics import roc_auc_score<compute_test_metric>
train['item_description'] = train['item_description'].fillna('No Description')
 test['item_description'] = test['item_description'].fillna('No Description' )
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<import_modules>
def split_cat(category_name):
try:
return category_name.split('/')
 return("done")
 except:
return ['Null', 'Null', 'Null']
from torchvision.models import *<choose_model_class>
train['cat_1'], train['cat_2'], train['cat_3'] = zip(*train['category_name'].apply(lambda x:split_cat(x)))
 test['cat_1'], test['cat_2'], test['cat_3'] = zip(*test['category_name'].apply(lambda x: split_cat(x)))
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)<define_search_space>
( train['shipping'].value_counts())*100/train.shape[0]
lr = 2e-03<train_model>
brands =train['brand_name'].value_counts()
print(brands[:10] )
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()<find_best_params>
print("Có %d nhãn ở cột cat_1." % train['cat_1'].nunique() )
learn.unfreeze()
learn.lr_find()<train_model>
print("Có %d nhãn ở cột cat_2." % train['cat_2'].nunique() )
learn.fit_one_cycle(1,slice(1e-4,1e-3))<compute_test_metric>
print("Có %d nhãn ở cột cat_3." % train['cat_3'].nunique() )
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score<compute_test_metric>
train['name'].value_counts() [:10]
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta<predict_on_test>
lb_item_condition_id = LabelBinarizer(sparse_output=True)
 train_condition = lb_item_condition_id.fit_transform(train['item_condition_id'])
 test_condition = lb_item_condition_id.transform(test['item_condition_id'] )
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
lb_shipping = LabelBinarizer(sparse_output=True)
 train_shipping = lb_shipping.fit_transform(train['shipping'])
 test_shipping = lb_shipping.transform(test['shipping'] )
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )<load_from_csv>
lb_brand_name = LabelBinarizer(sparse_output=True)
 train_brand_name= lb_brand_name.fit_transform(train['brand_name'])
 test_brand_name = lb_brand_name.transform(test['brand_name'])
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()<data_type_conversions>
count_vec = CountVectorizer()
train_name = count_vec.fit_transform(train['name'])
 test_name = count_vec.transform(test['name'] )
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )<save_to_csv>
tfidf_des = TfidfVectorizer(max_features=50000, ngram_range=(1, 3), stop_words='english')
 train_des = tfidf_des.fit_transform(train['item_description'])
 test_des = tfidf_des.transform(test['item_description'] )
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )<save_to_csv>
lb_cat_1 = LabelBinarizer(sparse_output=True)
 train_cat_1 = lb_cat_1.fit_transform(train['cat_1'])
 test_cat_1 = lb_cat_1.transform(test['cat_1'])
 lb_cat_2 = LabelBinarizer(sparse_output=True)
 train_cat_2 = lb_cat_2.fit_transform(train['cat_2'])
 test_cat_2 = lb_cat_2.transform(test['cat_2'])
 lb_cat_3 = LabelBinarizer(sparse_output=True)
 train_cat_3 = lb_cat_3.fit_transform(train['cat_3'])
 test_cat_3 = lb_cat_3.transform(test['cat_3'] )
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )<set_options>
from scipy.sparse import hstack
import gc
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload<define_variables>
sparse_matrix_list =(train_name, train_des, train_brand_name, train_condition,
train_shipping, train_cat_1, train_cat_2, train_cat_3 )
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96<load_from_csv>
def rmsle(y, y_preds):
assert len(y)== len(y_preds)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y_preds), 2)) )
df_trn=pd.read_csv(train_lbl )<feature_engineering>
def run_model(model, matrix_list):
X = hstack(matrix_list ).tocsr()
X_train, x_test, Y_train, y_test = train_test_split(X, np.log1p(train['price']), test_size=0.2)
 model.fit(X_train, Y_train)
 preds = model.predict(x_test)
 del X, X_train, x_test, Y_train
gc.collect()
return preds, y_test
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)<load_from_csv>
model = ridge = Ridge()
Ridge_preds, y_test = run_model(model, matrix_list=sparse_matrix_list)
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )<define_variables>
print("rmsle: "+str(rmsle(np.expm1(y_test), np.expm1(Ridge_preds))))
data.show_batch(rows=5, figsize=(12,9))<import_modules>
lgbm_reg = LGBMRegressor()
lgbm_preds, y_test = run_model(model=lgbm_reg, matrix_list=sparse_matrix_list )
from sklearn.metrics import roc_auc_score<compute_test_metric>
print("rmsle: "+str(rmsle(np.expm1(y_test), np.expm1(lgbm_preds))))
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<import_modules>
sparse_matrix_list =(train_name, train_des, train_brand_name, train_condition,
train_shipping, train_cat_1, train_cat_2, train_cat_3)
 X_train = hstack(sparse_matrix_list ).tocsr()
X_train
from torchvision.models import *<choose_model_class>
y_train = np.log1p(train['price'])
 y_train
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)<define_search_space>
Ridge_model = Ridge()
Ridge_model.fit(X_train, y_train )
lr = 1e-04<train_model>
sparse_matrix_list =(test_name, test_des, test_brand_name, test_condition,
test_shipping, test_cat_1, test_cat_2, test_cat_3)
 X_test = hstack(sparse_matrix_list ).tocsr()
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()<find_best_params>
preds = Ridge_model.predict(X_test)
 preds
learn.unfreeze()
learn.lr_find()<train_model>
preds = np.expm1(preds)
 preds
learn.fit_one_cycle(10,slice(1e-4,1e-3))<compute_test_metric>
submission = pd.read_csv('sample_submission_stg2.csv')
 submission
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score<compute_test_metric>
submission.loc[:, 'price'] = preds
submission
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta<predict_on_test>
submission.to_csv('submission.csv', index=False )
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
submission.to_csv('submission.csv', index=False )