Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<load_from_csv>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z
df_train = pd.read_csv(os.path.join(DIR_INPUT,"train.csv"))
df_train["pl"] = np.zeros_like(df_train["image_id"])
 df_train = pd.concat([df_train, pl_df] ).reset_index()<categorify>
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip
class CassavaDataset2(Dataset):
def __init__(self, df, dataset='train', transforms=None):
self.df = df
self.transforms=transforms
self.dataset=dataset
def __len__(self):
return self.df.shape[0]
def __getitem__(self, idx):
imageid = self.df.loc[idx, "image_id"]
label = self.df.loc[idx, "label"]
dir = self.df.loc[idx, "pl"]
if dir:
image_src = f'{DIR_INPUT}/test_images/{imageid}'
else:
image_src = f'{DIR_INPUT}/train_images/{imageid}'
image = cv2.imread(image_src)
 if self.transforms:
transformed = self.transforms(image=image)
 images = transformed['image']
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images), torch.tensor(label)
 transforms_train = albumentations.Compose([
albumentations.ShiftScaleRotate(scale_limit=0.3, rotate_limit=180,p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.5),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.5),
],p=0.9),
A.Cutout(num_holes=12, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
albumentations.Rotate(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.Resize(SIZE, SIZE, p=1.0),
])
 dataset_train = CassavaDataset2(df=df_train, dataset='test', transforms=transforms_train)
 dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)
 criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters() , lr=init_lr)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs )<normalization>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import math
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
scaler = torch.cuda.amp.GradScaler(enabled=False)
 def train_epoch(loader, optimizer):
model.train()
train_loss = []
bar = tqdm(loader)
 i = 0
for(data, target)in bar:
data, target = data.to(device), target.to(device ).long()
loss_func = criterion
optimizer.zero_grad()
with torch.cuda.amp.autocast(enabled=False):
logits = model(data ).squeeze(1)
 loss = loss_func(logits, target)
 scaler.scale(loss ).backward()
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
loss_np = loss.detach().cpu().numpy()
train_loss.append(loss_np)
 smooth_loss = sum(train_loss[-100:])/ min(len(train_loss), 100)
 bar.set_description('loss: %.5f, smth: %.5f' %(loss_np, smooth_loss))
i+=1
if commit and i==10:
break
return np.mean(train_loss )<train_model>
train = pd.read_table('train.tsv')
 test = pd.read_table('test_stg2.tsv')
 print(train.shape)
 print(test.shape )
for epoch in range(1, n_epochs+1):
torch.cuda.empty_cache()
scheduler.step(epoch-1)
 train_loss = train_epoch(dataloader_train , optimizer )<choose_model_class>
train.isnull().sum()
submissions = None
device = torch.device("cuda:0")if torch.cuda.is_available() else torch.device('cpu')
 for i_fold in range(N_FOLDS):
model.eval()
transforms = tta.Compose(
 [
tta.HorizontalFlip() ,
] 
)
 tta_models = []
for model in [model]:
tta_models.append(tta.ClassificationTTAWrapper(model, transforms))
for net in tta_models:
test_preds = None
for step, batch in enumerate(dataloader_test):
images = batch
images = images.to(device, dtype=torch.float)
 with torch.no_grad() :
outputs = net(images)
 preds = torch.softmax(outputs, dim=1 ).data.cpu()
if test_preds is None:
test_preds = preds
else:
test_preds = torch.cat(( test_preds, preds), dim=0)
 if submissions is None:
submissions = test_preds
else:
submissions += test_preds 
<save_to_csv>
def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset )
submission_df['label'] = torch.argmax(submissions, dim=1)
 submission_df.to_csv('submission.csv', index=False)
 submission_df<install_modules>
train = handle_missing(train)
 test = handle_missing(test )
!pip install.. /input/vision-transformer/vision_transformer_pytorch-1.0.2-py2.py3-none-any.whl<load_pretrained>
def transform_category_name(category_name):
try:
sub1, sub2, sub3 = category_name.split('/')
 return sub1, sub2, sub3
except:
return "none", "none", "none"
train['subcat_1'], train['subcat_2'], train['subcat_3'] = zip(*train['category_name'].apply(transform_category_name))
test['subcat_1'], test['subcat_2'], test['subcat_3'] = zip(*test['category_name'].apply(transform_category_name))
train.head(10 )
model = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 model.load_state_dict(torch.load('.. /input/vitb16trained/ViT-B_16_trained.pt',map_location=torch.device('cpu')) )<load_from_csv>
print("There are %d unique subcat_1." % train['subcat_1'].nunique())
 print("There are %d unique subcat_2." % train['subcat_2'].nunique())
 print("There are %d unique subcat_3." % train['subcat_3'].nunique() )
test_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 image_path = ".. /input/cassava-leaf-disease-classification/test_images/"
test_targets = test_df.label.values
test_aug = albu.Compose([
albu.CenterCrop(512, 512, p=1.) ,
albu.Resize(384, 384),
albu.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0),
ToTensorV2() ], p=1.)
test_dataset=CassavaDataset(
 df=test_df,
imfolder=image_path,
train=False,
transforms=test_aug 
)
 test_loader = DataLoader(
 test_dataset,
batch_size=4,
num_workers=4,
shuffle=False,
)<categorify>
train['category_name'].value_counts() [:10]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 predictions=[]
for imgs in test_loader:
imgs = imgs.to(device)
 with torch.no_grad() :
model=model.to(device)
 outputs = model(imgs)
 _, predicted = torch.max(outputs, dim=1)
 predicted=predicted.to('cpu')
 predictions.append(predicted )<save_to_csv>
lb_item_condition_id = LabelBinarizer(sparse_output=True)
 train_condition = lb_item_condition_id.fit_transform(train['item_condition_id'])
 test_condition = lb_item_condition_id.transform(test['item_condition_id'])
 train_condition.shape
test_df['label'] = np.concatenate(predictions)
 test_df.to_csv('submission.csv', index=False )<define_variables>
lb_shipping = LabelBinarizer(sparse_output=True)
 train_shipping = lb_shipping.fit_transform(train['shipping'])
 test_shipping = lb_shipping.transform(test['shipping'])
 train_shipping.shape
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
lb_brand_name = LabelBinarizer(sparse_output=True)
 train_brand_name= lb_brand_name.fit_transform(train['brand_name'])
 test_brand_name = lb_brand_name.transform(test['brand_name'])
 train_brand_name.shape
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
count_vec = CountVectorizer()
train_name = count_vec.fit_transform(train['name'])
 test_name = count_vec.transform(test['name'])
 print(train_name.shape )
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 512,
'epochs': 32,
'train_bs': 28,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 1,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
tfidf_des = TfidfVectorizer(max_features=50000, ngram_range=(1, 3), stop_words='english')
 train_des = tfidf_des.fit_transform(train['item_description'])
 test_des = tfidf_des.transform(test['item_description'])
 train_des.shape
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
lb_cat_1 = LabelBinarizer(sparse_output=True)
 train_cat_1 = lb_cat_1.fit_transform(train['subcat_1'])
 test_cat_1 = lb_cat_1.transform(test['subcat_1'])
 lb_cat_2 = LabelBinarizer(sparse_output=True)
 train_cat_2 = lb_cat_2.fit_transform(train['subcat_2'])
 test_cat_2 = lb_cat_2.transform(test['subcat_2'])
 lb_cat_3 = LabelBinarizer(sparse_output=True)
 train_cat_3 = lb_cat_3.fit_transform(train['subcat_3'])
 test_cat_3 = lb_cat_3.transform(test['subcat_3'] )
train.label.value_counts()<load_from_csv>
sparse_matrix_list =(train_name, train_des, train_brand_name, train_condition,
train_shipping, train_cat_1, train_cat_2, train_cat_3)
 X_train = hstack(sparse_matrix_list ).tocsr()
print(X_train.shape)
 del X_train
gc.collect()
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<categorify>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
def run_model(model, matrix_list):
X = hstack(matrix_list ).tocsr()
X_train, x_test, Y_train, y_test = train_test_split(X, np.log1p(train['price']), test_size=0.2)
 model.fit(X_train, Y_train)
 preds = model.predict(x_test)
 del X, X_train, x_test, Y_train
gc.collect()
return preds, y_test
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
model = ridge = Ridge()
Ridge_preds, y_test = run_model(model, matrix_list=sparse_matrix_list)
 print("rmsle: "+str(rmsle(np.expm1(y_test), np.expm1(Ridge_preds))))
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<split>
submission = pd.read_csv('sample_submission_stg2.csv')
 submission
submission.loc[:, 'price'] = preds
submission
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()<feature_engineering>
submission.to_csv('submission.csv', index=False )
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
submission.to_csv('submission.csv', index=False )