Kaggle competition: Mercari Price Suggestion Challenge
%%time
lr = LogisticRegression(C=0.002, max_iter=10000)
 lr.fit(X_train, y_train )<predict_on_test>
import pandas as pd
import numpy as np
import scipy
import time
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from scipy.sparse import csr_matrix, hstack
val_preds = lr.predict_proba(X_val)[:,1]<compute_test_metric>
np.random.seed(7)
 NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 50000
start_time = time.time()
tcurrent = start_time
roc_auc_score(y_val, val_preds )<compute_test_metric>
def rmsle(y, y0):
return np.sqrt(np.mean(np.square(y - y0)))
 def handle_missing_inplace(dataset):
dataset['category_name'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category = dataset['category_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['category_name'].isin(pop_category), 'category_name'] = 'missing'
def to_categorical(dataset):
dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['brand_name'] = dataset['brand_name'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category' )
log_loss(y_val, val_preds )<predict_on_test>
train = pd.read_table('.. /input/train.tsv', engine='c')
 test = pd.read_table('.. /input/test.tsv', engine='c')
 print('[{}] Finished to load data'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape )
test_preds = lr.predict_proba(test_DenseNet201)[:,1]<save_to_csv>
nrow_train = train.shape[0]
y = np.log1p(train["price"])
 merge = pd.concat([train, test])
 submission = test[['test_id']]
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()<import_modules>
handle_missing_inplace(merge)
 print('[{}] Finished to handle missing'.format(time.time() - start_time))
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
cutting(merge)
 print('[{}] Finished to cut'.format(time.time() - start_time))
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
to_categorical(merge)
 print('[{}] Finished to convert categorical'.format(time.time() - start_time))
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
cv = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = cv.fit_transform(merge['name'])
 print('[{}] Finished count vectorize `name`'.format(time.time() - start_time))
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
cv = CountVectorizer()
X_category = cv.fit_transform(merge['category_name'])
 print('[{}] Finished count vectorize `category_name`'.format(time.time() - start_time))
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(2, 2),
stop_words='english')
 X_description = tv.fit_transform(merge['item_description'])
 print('[{}] Finished TFIDF vectorize `item_description`'.format(time.time() - start_time))
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(merge['brand_name'])
 print('[{}] Finished label binarize `brand_name`'.format(time.time() - start_time))
train_DenseNet121 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_2/train_DenseNet121.npy')
 test_DenseNet121 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_2/test_DenseNet121.npy' )<split>
X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']],
sparse=True ).values)
 print('[{}] Finished to get dummies on `item_condition_id` and `shipping`'.format(time.time() - start_time))
X_train, X_val, y_train, y_val = train_test_split(train_DenseNet121, target, test_size=0.1, random_state=42 )<train_model>
sparse_merge = hstack(( X_dummies, X_description, X_brand, X_category, X_name)).tocsr()
print('[{}] Finished to create sparse merge'.format(time.time() - start_time))
%%time
lr = LogisticRegression(C=0.002, max_iter=10000)
 lr.fit(X_train, y_train )<predict_on_test>
X = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_train:]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=7 )
val_preds = lr.predict_proba(X_val)[:,1]<compute_test_metric>
model = Ridge(solver="sag", fit_intercept=True, random_state=7, alpha=3)
 model.fit(X_train, y_train)
 print('[{}] Finished to train ridge sag'.format(time.time() - start_time))
preds = model.predict(X=X_test)
 print('[{}] Finished to predict ridge sag'.format(time.time() - start_time))
roc_auc_score(y_val, val_preds )<compute_test_metric>
print('RMSLE: {}'.format(rmsle(y_val, model.predict(X_val))))
<predict_on_test><EOS> 
submission['price'] = np.expm1(preds)
 submission.to_csv("sample_submission.csv", index=False )