Kaggle competition: Mercari Price Suggestion Challenge
res50 = pd.read_csv(".. /input/cancer-resnet50-v2-for-ensemble/validation_0.9727705717086792.csv")
 res50_test = pd.read_csv(".. /input/cancer-resnet50-v2-for-ensemble/submission_0.9727705717086792.csv" )<feature_engineering>
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip
<feature_engineering>
train_dataset = pd.read_table("train.tsv")
 test_dataset = pd.read_table("test_stg2.tsv" )
def softmax_df(df, model_name, test=False):
if test:
df[model_name+'_0'] = np.exp(df['pred_0'])
 df[model_name+'_1'] = np.exp(df['pred_1'])
 else:
df[model_name+'_0'] = np.exp(df['val_0'])
 df[model_name+'_1'] = np.exp(df['val_1'])
 df[model_name+'sum'] = df[model_name+'_0'] + df[model_name+'_1']
df[model_name+'softmax'] = df[model_name+'_1'] / df[model_name+'sum']
return df[model_name+'softmax']<create_dataframe>
train_dataset = train_dataset[train_dataset['price'] > 0].reset_index(drop = True)
 train, validation = train_test_split(train_dataset, test_size = 0.1, random_state = 30)
 test = test_dataset.copy()
print(train.shape)
 print(validation.shape)
 print(test.shape )
<create_dataframe>
train.isnull().sum()
train = pd.DataFrame({'dense161_0':dense161.val_0, 'dense161_1':dense161.val_1,
'dense201_0':dense201.val_0, 'dense201_1':dense201.val_1,
'res50_0':res50.val_0, 'res50_1':res50.val_1,
"y":dense161.ground_truth_label})
 test = pd.DataFrame({'dense161_0':dense161_test.pred_0, 'dense161_1':dense161_test.pred_1,
'dense201_0':dense201_test.pred_0, 'dense201_1':dense201_test.pred_1,
'res50_0':res50_test.pred_0, 'res50_1':res50_test.pred_1})
 test.y=0<compute_test_metric>
train.isnull().sum()
def roc_score(inp, target):
_, indices = inp.max(1)
 return torch.Tensor([roc_auc_score(target, indices)])[0]<choose_model_class>
validation.isnull().sum()
learn = tabular_learner(data, layers=[10, 10, 10], metrics=[accuracy, roc_score], ps=0.5, wd=1e-1, model_dir='./' ).to_fp16()<train_on_grid>
test.isnull().sum()
ES = EarlyStoppingCallback(learn, monitor='roc_score',patience = 5)
 RLR = ReduceLROnPlateauCallback(learn, monitor='roc_score',patience = 2)
 SAVEML = SaveModelCallback(learn, every='improvement', monitor='roc_score', name='best')
 learn.fit_one_cycle(20, 1e-3, callbacks = [ES, RLR, SAVEML] )<load_pretrained>
test.isnull().sum()
learn.load('best' )<predict_on_test>
def fill(data):
data['category_name'].fillna('Missing', inplace = True)
 data['brand_name'].fillna('Missing', inplace = True)
 data['item_description'].fillna('Missing', inplace = True)
 return data
preds, _ = learn.get_preds(DatasetType.Test)
 preds = torch.softmax(preds, dim=1)[:, 1].numpy()<load_from_csv>
fill(train)
 fill(validation)
 fill(test )
sub = pd.read_csv(".. /input/histopathologic-cancer-detection/sample_submission.csv")
 sub.head()<prepare_output>
def split_category(cate):
try:
sub_cate1, sub_cate2, sub_cate3 = cate.split("/")
 return sub_cate1, sub_cate2, sub_cate3
except:
return("Missing", "Missing", "Missing")
 def creat_category(data):
data['sub_category1'], data['sub_category2'], data['sub_category3'] = zip(*data['category_name'].apply(lambda x: split_category(x)) )
sub['label'] = preds<save_to_csv>
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from scipy.sparse import csr_matrix, hstack
sub.to_csv(f'submission_{auc_val}.csv', header=True, index=False )<set_options>
merge: pd.DataFrame = pd.concat([train, validation, test])
 submission: pd.DataFrame = test[['test_id']]
n_train = train.shape[0]
n_validation = validation.shape[0]
n_test = test.shape[0]
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload<define_variables>
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 30000
def condition2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def shipping2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def brandname2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def name2vec(data):
vectorizer = CountVectorizer(
 min_df=NAME_MIN_DF,
ngram_range=(1, 2),
stop_words='english' 
)
 vec = vectorizer.fit_transform(data)
 return vec
def description2vec(data):
vectorizer = TfidfVectorizer(
 max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(1, 2),
stop_words='english' 
)
 vec = vectorizer.fit_transform(data)
 return vec
def category2vec(data):
vectorizer = CountVectorizer()
data_vec = []
for x in data:
category_vec = vectorizer.fit_transform(x)
 data_vec.append(category_vec)
 print('Convert Category ')
 return hstack(data_vec)
 def put_all_together(merge):
res = hstack((
name2vec(merge['name']),
condition2vec(merge['item_condition_id']),
brandname2vec(merge['brand_name']),
shipping2vec(merge['shipping']),
description2vec(merge['item_description']),
category2vec(( merge['sub_category1'], merge['sub_category2'], merge['sub_category3'])) 
))
return res
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96<load_from_csv>
all_data = all_data.tocsr()
train_X, validation_X, test_X = all_data[:n_train], all_data[n_train:n_train + n_validation], all_data[n_train + n_validation:]
df_trn=pd.read_csv(train_lbl )<feature_engineering>
train_Y = np.log1p(train["price"])
 validation_Y = np.log1p(validation["price"] )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)<load_from_csv>
from sklearn.linear_model import Ridge
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )<define_variables>
model = Ridge(
 alpha=.5,
copy_X=True,
fit_intercept=True,
max_iter=100,
normalize=False,
random_state=101,
solver='auto',
tol=0.01 
)
 model.fit(train_X, train_Y )
data.show_batch(rows=5, figsize=(12,9))<import_modules>
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)) )
from sklearn.metrics import roc_auc_score<compute_test_metric>
val_pred = model.predict(validation_X)
 val_pred = np.expm1(val_pred)
 print(rmsle(val_pred, validation["price"]))
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<import_modules>
test_pred = model.predict(test_X)
 submission['price'] = np.expm1(test_pred )
<choose_model_class><EOS> 
submission.to_csv("submission.csv", index=False )