Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<define_search_space>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
lr = 1e-04<train_model>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()<find_best_params>
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip
learn.unfreeze()
learn.lr_find()<train_model>
train_data = pd.read_csv('train.tsv', sep='\t')
 train_data.head(5 )
learn.fit_one_cycle(1,slice(1e-4,1e-3))<predict_on_test>
train_data.isnull().sum()
m = learn.model.eval()<define_variables>
train_data = train_data[train_data['price'] > 0].reset_index(drop=True)
 train_data,validation_data=train_test_split(train_data,test_size=0.1,random_state=42)
 print(train_data.shape)
 print(validation_data.shape )
idx = 4<import_modules>
train_data.isnull().sum()
from fastai.callbacks.hooks import *<categorify>
validation_data.isnull().sum()
def hooked_backward(cat=y):
with hook_output(m[0])as hook_a:
with hook_output(m[0], grad = True)as hook_g:
preds = m(xb)
 preds[0,int(cat)].backward()
return hook_a, hook_g 
<compute_test_metric>
train = train_data.copy()
valid = validation_data.copy()
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score<compute_test_metric>
def split_categories(category):
try:
sub_category1,sub_category2,sub_category3 = category.split("/")
 return sub_category1,sub_category2,sub_category3
except:
return("No label","No label","No label")
 def create_split_categories(data):
data['sub_category1'],data['sub_category2'],data['sub_category3']=zip(*data['category_name'].\
apply(lambda x: split_categories(x)) )
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta<predict_on_test>
create_split_categories(train_data)
 create_split_categories(validation_data )
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
def fill_missing_values(data):
data['category_name'].fillna('unknown_cat', inplace=True)
 data['brand_name'].fillna('unknown_brand', inplace=True)
 data['item_description'].fillna('unknown_description', inplace=True)
 return data
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )<load_from_csv>
fill_missing_values(train_data)
 fill_missing_values(validation_data )
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()<data_type_conversions>
test_data = pd.read_csv('test_stg2.tsv',sep='\t')
 test = test_data.copy()
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )<save_to_csv>
test_data.isnull().sum()
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )<save_to_csv>
create_split_categories(test_data)
 fill_missing_values(test_data )
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )<load_from_csv>
train_data['log_prices']= np.log(train_data['price']+1 )
labels = pd.read_csv('.. /input/histopathologic-cancer-detection/train_labels.csv')
 tr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.07, random_state=SEED)
 img_class_dict = {k:v for k, v in zip(labels.id, labels.label)}
class CancerDataset(Dataset):
def __init__(self, datafolder, datatype='train', idx=[], transform = transforms.Compose([transforms.CenterCrop(48),transforms.ToTensor() ]), labels_dict={}):
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [s for s in os.listdir(datafolder)]
self.image_files_list = [self.image_files_list[i] for i in idx]
self.transform = transform
self.labels_dict = labels_dict
if self.datatype == 'train':
self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]
else:
self.labels = [0 for _ in range(len(self.image_files_list)) ]
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 img = cv2.imread(img_name)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 image = self.transform(image=img)
 image = image['image']
img_name_short = self.image_files_list[idx].split('.')[0]
if self.datatype == 'train':
label = self.labels_dict[img_name_short]
else:
label = 0
return image, label
data_transforms = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.OneOf([
albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen() , albumentations.IAAEmboss() ,
albumentations.RandomBrightness() , albumentations.RandomContrast() ,
albumentations.JpegCompression() , albumentations.Blur() , albumentations.GaussNoise() ], p=0.5),
albumentations.HueSaturationValue(p=0.5),
albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_test = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.Normalize() ,
AT.ToTensor()
])
 class Densenet169(nn.Module):
def __init__(self, pretrained=True):
super(Densenet169, self ).__init__()
self.model = models.densenet169(pretrained=pretrained)
 self.linear = nn.Linear(1000+2, 16)
 self.bn = nn.BatchNorm1d(16)
 self.dropout = nn.Dropout(0.2)
 self.elu = nn.ELU()
self.out = nn.Linear(16, 1)
 def forward(self, x):
out = self.model(x)
 batch = out.shape[0]
max_pool, _ = torch.max(out, 1, keepdim=True)
 avg_pool = torch.mean(out, 1, keepdim=True)
 out = out.view(batch, -1)
 conc = torch.cat(( out, max_pool, avg_pool), 1)
 conc = self.linear(conc)
 conc = self.elu(conc)
 conc = self.bn(conc)
 conc = self.dropout(conc)
 res = self.out(conc)
 return res
model_conv = Densenet169(pretrained=False)
 model_conv.load_state_dict(torch.load(".. /input/densenet169-pretrain-cancer/model"), strict=False)
 model_conv.eval().cuda()<categorify>
validation_data['log_prices']= np.log(validation_data['price']+1 )
for tta in range(num_tta):
seed_everything(SEED+10*tta)
 test_idx = [i for i in range(len(os.listdir(".. /input/histopathologic-cancer-detection/test")))]
test_set = CancerDataset(datafolder='.. /input/histopathologic-cancer-detection/test/', idx=test_idx, datatype='test', transform=data_transforms)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 preds = []
for batch_i,(data, target)in enumerate(test_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data ).detach()
pr = output[:,0].cpu().numpy()
for i in pr:
preds.append(i)
 test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 sub = pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv')
 sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')
 sub = sub[['id', 'preds']]
sub.columns = ['id', 'label']
sub.head()
sub.to_csv('single_model_'+str(tta)+'.csv', index=False)
 del model_conv
gc.collect()
torch.cuda.empty_cache()<feature_engineering>
nltk.download('stopwords' )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def sigmoid_pd(df):
df["label"] = df["label"].apply(sigmoid)
 return df<save_to_csv>
stop = stopwords.words('english')
 def remove_stop_words(x):
x = ' '.join([i for i in x.lower().split(' ')if i not in stop])
 return x
df = pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv')
 for tta in range(num_tta):
df0 = pd.read_csv('.. /working/single_model_'+str(tta)+'.csv')
 df0 = sigmoid_pd(df0)
 df['label'] += df0['label']
if(tta+1==8):
df_tmp = df.copy()
df_tmp['label'] /= 8
df_tmp.to_csv('submission_tta_'+str(tta+1)+'.csv',index=False)
 if(tta+1==16):
df_tmp = df.copy()
df_tmp['label'] /= 16
df_tmp.to_csv('submission_tta_'+str(tta+1)+'.csv',index=False)
 df['label'] /= num_tta
df.to_csv('submission_tta_'+str(num_tta)+'.csv',index=False )<set_options>
train_data['item_description'] = train_data['item_description'].apply(remove_stop_words)
 validation_data['item_description'] = validation_data['item_description'].apply(remove_stop_words)
 test_data['item_description'] = test_data['item_description'].apply(remove_stop_words )
warnings.simplefilter('ignore', category=FutureWarning)
 warnings.simplefilter('ignore', category=UserWarning)
 %matplotlib inline<load_from_csv>
, \\
def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
def text_preprocessing(text_col):
preprocessed_total = []
for sentence in tqdm(text_col.values):
sent = decontracted(sentence)
 sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 preprocessed_total.append(sent.lower().strip())
 return preprocessed_total
train_df = pd.read_csv(PATH/'train_labels.csv')
 print(train_df.shape)
 train_df.head()<count_values>
train_data['item_description']=text_preprocessing(train_data['item_description'])
 validation_data['item_description']=text_preprocessing(validation_data['item_description'])
 test_data['item_description']=text_preprocessing(test_data['item_description'])
 train_data['name']=text_preprocessing(train_data['name'])
 validation_data['name']=text_preprocessing(validation_data['name'])
 test_data['name']=text_preprocessing(test_data['name'] )
train_df['label'].value_counts(normalize=True )<load_from_csv>
print(train_data['item_description'].iloc[33],len(train_data['item_description'].iloc[33].split(' ')))
 print(train['item_description'].iloc[33],len(train['item_description'].iloc[33].split(' ')) )
src =(ImageItemList.from_csv(PATH, folder='train', csv_name='train_labels.csv', suffix='.tif')
.random_split_by_pct(0.1, seed=77)
.label_from_df() 
.add_test_folder() )<normalization>
def clean_cat(cat_col):
cat_list = []
for i in tqdm(cat_col.values):
i = re.sub('[^A-Za-z0-9]+', ' ', i)
 i = i.replace(' ','')
 i = i.replace('&','_')
 cat_list.append(i.strip())
 return cat_list
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0, max_zoom=1., max_lighting=0.05, max_warp=0)
 data =(src.transform(tfms, size=96, resize_method=ResizeMethod.SQUISH)
.databunch(bs=64, path='.'))
data.normalize(imagenet_stats);<define_variables>
train_data['sub_category1'] = clean_cat(train_data['sub_category1'])
 validation_data['sub_category1'] = clean_cat(validation_data['sub_category1'])
 test_data['sub_category1'] = clean_cat(test_data['sub_category1'])
 train_data['sub_category2'] = clean_cat(train_data['sub_category2'])
 validation_data['sub_category2'] = clean_cat(validation_data['sub_category2'])
 test_data['sub_category2'] = clean_cat(test_data['sub_category2'])
 train_data['sub_category3'] = clean_cat(train_data['sub_category3'])
 validation_data['sub_category3'] = clean_cat(validation_data['sub_category3'])
 test_data['sub_category3'] = clean_cat(test_data['sub_category3'] )
data.show_batch(rows=5, figsize=(15, 15))<compute_train_metric>
train_data['brand_name'] = clean_cat(train_data['brand_name'])
 validation_data['brand_name'] = clean_cat(validation_data['brand_name'])
 test_data['brand_name'] = clean_cat(test_data['brand_name'] )
class FocalLoss(nn.Module):
def __init__(self, alpha=1., gamma=1.) :
super().__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self, inputs, targets, **kwargs):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss = self.alpha *(( 1-pt)**self.gamma)* CE_loss
return F_loss.mean()
def roc_score(inp, target):
_, indices = inp.max(1)
 return torch.Tensor([roc_auc_score(target, indices)])[0]<choose_model_class>
countvectorizer=CountVectorizer().fit(train_data['sub_category1'])
 bow_cat1_train=countvectorizer.transform(train_data['sub_category1'])
 bow_cat1_val=countvectorizer.transform(validation_data['sub_category1'])
 bow_cat1_test=countvectorizer.transform(test_data['sub_category1'])
 print("Shape of sub_category1 features:")
 print(bow_cat1_train.shape)
 print(bow_cat1_val.shape)
 print(bow_cat1_test.shape)
 print("Some Features are:")
 print(countvectorizer.get_feature_names() )
loss_func = FocalLoss(gamma=1.)
learn = create_cnn(data, models.densenet121, metrics=[accuracy, roc_score], loss_func=loss_func )<find_best_params>
countvectorizer=CountVectorizer().fit(train_data['sub_category2'])
 bow_cat2_train=countvectorizer.transform(train_data['sub_category2'])
 bow_cat2_val=countvectorizer.transform(validation_data['sub_category2'])
 bow_cat2_test=countvectorizer.transform(test_data['sub_category2'])
 print("Shape of sub_category2 features:")
 print(bow_cat2_train.shape)
 print(bow_cat2_val.shape)
 print(bow_cat2_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [50:60] )
learn.lr_find()<train_model>
countvectorizer=CountVectorizer().fit(train_data['sub_category3'])
 bow_cat3_train=countvectorizer.transform(train_data['sub_category3'])
 bow_cat3_val=countvectorizer.transform(validation_data['sub_category3'])
 bow_cat3_test=countvectorizer.transform(test_data['sub_category3'])
("Shape of sub_category3 features:")
 print(bow_cat3_train.shape)
 print(bow_cat3_val.shape)
 print(bow_cat3_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [200:210] )
learn.fit_one_cycle(2, 1e-3 )<save_model>
countvectorizer=CountVectorizer().fit(train_data['brand_name'])
 bow_brand_train=countvectorizer.transform(train_data['brand_name'])
 bow_brand_val=countvectorizer.transform(validation_data['brand_name'])
 bow_brand_test=countvectorizer.transform(test_data['brand_name'])
("Shape of brand_name features:")
 print(bow_brand_train.shape)
 print(bow_brand_val.shape)
 print(bow_brand_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [35:45] )
learn.save('stage-1' )<load_pretrained>
countvectorizer=CountVectorizer(min_df=10 ).fit(train_data['name'])
 bow_name_train=countvectorizer.transform(train_data['name'])
 bow_name_val=countvectorizer.transform(validation_data['name'])
 bow_name_test=countvectorizer.transform(test_data['name'])
 print("After Vectorization of name feature: ")
 print(bow_name_train.shape)
 print(bow_name_val.shape)
 print(bow_name_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [210:220] )
learn.load('stage-1');<find_best_params>
tfidfvectorizer=TfidfVectorizer(ngram_range=(1,2),min_df=10,max_features=5000 ).fit(train_data['item_description'])
 tfidf_description_train=tfidfvectorizer.transform(train_data['item_description'])
 tfidf_description_val=tfidfvectorizer.transform(validation_data['item_description'])
 tfidf_description_test=tfidfvectorizer.transform(test_data['item_description'])
 print("After Vectorization of item description feature: ")
 print(tfidf_description_train.shape)
 print(tfidf_description_val.shape)
 print(tfidf_description_test.shape)
 print("Some Features are: ")
 print(tfidfvectorizer.get_feature_names() [222:234] )
learn.lr_find()<train_model>
features_train = csr_matrix(pd.get_dummies(train_data[['item_condition_id', 'shipping']],sparse=True ).values)
 features_val = csr_matrix(pd.get_dummies(validation_data[['item_condition_id', 'shipping']],sparse=True ).values)
 features_test = csr_matrix(pd.get_dummies(test_data[['item_condition_id', 'shipping']],sparse=True ).values)
 print(features_train.shape)
 print(features_val.shape)
 print(features_test.shape )
max_lr = 1e-4
learn.fit_one_cycle(4, slice(1e-6, max_lr))<save_model>
linearregression=LinearRegression(normalize=True)
 linearregression.fit(X_train,train_data['log_prices'])
 ytrain_predict=linearregression.predict(X_train)
 yval_predict=linearregression.predict(X_val)
 train_error=np.sqrt(mean_squared_log_error(train_data['log_prices'],ytrain_predict))
val_error=np.sqrt(mean_squared_log_error(validation_data['log_prices'],yval_predict))
print(" Linear Regression RMSLE on train is {} RMSLE on cv is {}".format(train_error,val_error))
learn.save('stage-2' )<load_pretrained>
yval_linear=linearregression.predict(X_val)
 ytest_linear=linearregression.predict(X_test )
learn.load('stage-2');<prepare_output>
submission_data = pd.read_csv('sample_submission_stg2.csv')
 submission_data.head(5 )
preds, y = learn.TTA(beta=0.4, ds_type=DatasetType.Test)
 preds = torch.softmax(preds, dim=1)[:, 1].numpy()<load_from_csv>
submission_data.loc[:, 'price'] = np.expm1(ytest_linear )
test_ids = [f.stem for f in learn.data.test_ds.items]
subm = pd.read_csv(PATH/'sample_submission.csv')
 orig_ids = list(subm['id'] )<save_to_csv>
submission_data.to_csv('submission.csv', index=False )