Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<set_options>
import pandas as pd
import operator
import matplotlib.pyplot as plt
import numpy as np
import nltk
import time
gc.collect()<choose_model_class>
df = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_sub = pd.read_csv('.. /input/test.tsv', sep='\t')
 submission = pd.DataFrame()
submission['test_id'] = df_sub.test_id.copy()
y_target = list(df.price )
class NNModel(object):
def __init__(self, model, target_variable='target'):
self.model = model
self.es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=EARLY_STOPPING, verbose=False, mode='auto', restore_best_weights=True)
 self.rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-6, mode='auto', verbose=False)
 self.target_variable = target_variable
self.scaling_stats = {}
self.predictors = ['bland-lgbt-folds_version_1', 'bland-lgbt-folds_version_2',
'pp-lgbt-refactored_version_7',
'bland-lgbt-on-pp-leaks-train-fe_version1',
'bland-nn-on-pp-leaks-train-fe_version1',
'Bland LGBT on PP Leaks Train FE_version4',
'bland-nn-on-pp-leaks-train-fe_version2',
'Bland LGBT on PP Leaks Train FE_version2']
def train_preprocessing(self, data):
y = data[self.target_variable]
data = data[self.predictors]
print(data.shape)
 for col in self.predictors:
self.scaling_stats[col] = {'mean':data[col].mean() , 'std':data[col].std() }
data[col] =(data[col] - self.scaling_stats[col]['mean'])/ self.scaling_stats[col]['std']
print(data.isna().sum().sum())
 print('Scaling completed!')
 gc.collect()
return data.values, y.values
def test_preprocessing(self, data, is_val=False):
if is_val:
y = data[self.target_variable]
data = data[self.predictors]
gc.collect()
for col in self.predictors:
data[col] =(data[col] - self.scaling_stats[col]['mean'])/ self.scaling_stats[col]['std']
gc.collect()
if is_val:
return data.values, y.values
else:
return data.values
def fit(self, data, data_val):
data, y_train = self.train_preprocessing(data)
 data_val, y_val = self.test_preprocessing(data_val, is_val=True)
 self.model.fit(
 data, y_train, epochs=N_EPOCHS, batch_size=BATCH_SIZE, validation_data=(data_val, y_val), verbose=True, callbacks=[self.es, self.rlr] 
)
 def predict(self, data):
data = self.test_preprocessing(data, is_val=False)
 return self.model.predict(data, batch_size=BATCH_SIZE, verbose=True ).flatten()<train_model>
def null_percentage(column):
df_name = column.name
nans = np.count_nonzero(column.isnull().values)
 total = column.size
frac = nans / total
perc = int(frac * 100)
 print('%d%% or %d missing from %s column.' % 
(perc, nans, df_name))
def check_null(df, columns):
for col in columns:
null_percentage(df[col])
 check_null(df, df.columns)
def time_val(data, model_create_func, metric_to_use=mean_squared_error, target_var_name='target', test_to_predict=None):
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 print('Starting Validation')
 results = []
data['pred'] = 0
if test_to_predict is not None:
test_prediction = []
for train_idx, test_idx in kf.split(data):
print('New Itter')
 model = model_create_func()
model.fit(data.iloc[train_idx].reset_index(drop=True),
data.iloc[test_idx].reset_index(drop=True))
gc.collect()
data['pred'].iloc[test_idx] = model.predict(data.iloc[test_idx].reset_index(drop=True))
gc.collect()
itter_metric = metric_to_use(data.iloc[test_idx][target_var_name], data['pred'].iloc[test_idx])
 print('Itter metric: '+str(itter_metric))
results.append(itter_metric)
 gc.collect()
if test_to_predict is not None:
test_prediction.append(model.predict(test_to_predict))
gc.collect()
if test_to_predict is not None:
return results, sum(test_prediction)/NUM_FOLDS
else:
return results 
<choose_model_class>
def merc_imputer(df_temp):
df_temp.brand_name = df_temp.brand_name.replace(np.nan, 'no_brand')
 df_temp.category_name = df_temp.category_name.replace(np.nan, 'uncategorized/uncategorized')
 df_temp.item_description = df_temp.item_description.replace(np.nan, 'No description yet')
 df_temp.item_description = df_temp.item_description.replace('No description yet', 'no_description')
 return df_temp
df = merc_imputer(df)
 df_sub = merc_imputer(df_sub )
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 def create_model(inp_dim):
inps = Input(shape=(inp_dim,))
x = Dense(1 )(inps)
 model = Model(inputs=inps, outputs=x)
 model.compile(
 optimizer=Nadam(lr=1e-3),
loss=root_mean_squared_error 
)
 return model<set_options>
print('Training Data')
 check_null(df, df.columns)
 print('Submission Data')
 check_null(df_sub, df_sub.columns )
gc.collect()<choose_model_class>
df.shipping.value_counts()
if USE_STACKING:
NUM_FOLDS = 5
N_EPOCHS = 3
BATCH_SIZE = 256
EARLY_STOPPING = 2
my_model_create_f = lambda : NNModel(model = create_model(8))<feature_engineering>
print('$1 items: ' + str(df.price[df.price == 1].count()))
print('$2 items: ' + str(df.price[df.price == 2].count()))
print('$3 items: ' + str(df.price[df.price == 3].count()))
if USE_STACKING:
rf_res, test['target'] = time_val(train, my_model_create_f, test_to_predict=test )<compute_test_metric>
print('Free items: %d, representing %.5f%% of all items.' % 
(df.price[df.price == 0].count() , 
(df.price[df.price == 0].count() / df.shape[0])) )
if USE_STACKING:
print('Result: {} +/- {}'.format(round(np.mean(rf_res),5), round(np.std(rf_res),5)))
 print(mean_squared_error(train['pred'], train['target']))<feature_engineering>
print('Free items where seller pays shipping: %d.' %
df.price[operator.and_(df.price == 0, df.shipping == 1)].count() )
if USE_STACKING:
train = train.rename(columns={'pred':'meter_reading'})
 test = test.rename(columns={'target':'meter_reading'})
 gc.collect()
sub = pd.concat([train['meter_reading'], test['meter_reading']], axis=0)
 sub = sub.reset_index()
sub['meter_reading'] = np.expm1(sub['meter_reading'])
 sub.loc[sub['meter_reading'] < 0, 'meter_reading'] = 0
sub = sub.sort_values('row_id' )<save_to_csv>
print('No description:', str(df.item_description[df.item_description == 'no_description'].count()))
print('Uncategorized:',str(df.category_name[df.category_name == 'uncategorized/uncategorized'].count()))
if USE_GENETIC_ALG or USE_HYPEROPT:
all_sumbissions['meter_reading'] = all_sumbissions.iloc[:,best_cols].median(axis=1)
 all_sumbissions = all_sumbissions[['row_id','meter_reading']]
gc.collect()
all_sumbissions[['row_id','meter_reading']].to_csv('submission_blend.csv', index=False)
 elif USE_STACKING:
sub.to_csv('submission_blend.csv', index=False)
 else:
raise ValueError('Not ready' )<import_modules>
print('The top 75%% of categories represent %.1f%% of the dataset, and the top 50%% represent %.1f%%.' % 
(( sum([count for count in cat_counts if count > 10])/ len(df)) *100, 
(sum([count for count in cat_counts if count > 76])/ len(df)) *100))
import pandas as pd
import numpy as np<import_modules>
brand_counts = np.sort(df.brand_name.value_counts())
 print(str(len(brand_counts)) + ' brands total.')
 print(str(df.shape[0])+ ' records total.')
 print('Category frequency percentiles, marked by lines:
25%%: %d, 50%%: %d, 75%%: %d, 95%%: %d, 97.5%%: %d.' % 
(brand_counts[int(len(brand_counts)*0.25)],
brand_counts[int(len(brand_counts)*0.5)],
brand_counts[int(len(brand_counts)*0.75)],
brand_counts[int(len(brand_counts)*0.9)],
brand_counts[int(len(brand_counts)*0.95)]))
title = 'Brand Quantity ECDF Without Top 25 Outliers'
plt.figure(title, figsize=(30,10))
plt.title(title, fontsize=32)
 x = np.sort(df.brand_name.value_counts())
 x = x[0:-25]
y = np.arange(1, len(x)+ 1)/ len(x)
 plt.plot(x, y, marker='.', linestyle='none')
 plt.xticks(fontsize=24)
 plt.yticks(fontsize=26)
 plt.axvline(x=x[int(len(x)*0.25)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.5)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.75)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.95)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.975)], linewidth=1, color='b')
 plt.show()
import pandas as pd
import numpy as np<load_from_csv>
print('The top 75%% of categories represent %.1f%% of the dataset, and the top 50%% represent %.1f%%.' % 
(( sum([count for count in brand_counts if count > 1])/ len(df)) *100, 
(sum([count for count in brand_counts if count > 4])/ len(df)) *100))
fold_1 = pd.read_csv('/kaggle/input/lgbt-on-leaks-fold-1/submission.csv')
 fold_2 = pd.read_csv('/kaggle/input/lgbt-on-leaks-fold-2/submission.csv')
 fold_3 = pd.read_csv('/kaggle/input/lgbt-on-leaks-fold-3/submission.csv')
 fold_4 = pd.read_csv('/kaggle/input/lgbt-on-leaks-fold-4/submission.csv')
 fold_5 = pd.read_csv('/kaggle/input/lgbt-on-leaks-fold-5/submission.csv' )<feature_engineering>
print('%d items, or %.2f%%, are missing a brand name.' % 
(len(df[df.brand_name == 'no_brand']),
len(df[df.brand_name == 'no_brand'])/ len(df)) )
fold_1['meter_reading'] =(fold_1['meter_reading'] + fold_2['meter_reading'] + fold_3['meter_reading'] + fold_4['meter_reading'] + fold_5['meter_reading'])/5<feature_engineering>
nltk.data.path.append(r'D:\Python\Data Sets
ltk_data')
fold_1['meter_reading'] = np.expm1(fold_1['meter_reading'])
 fold_1.loc[fold_1['meter_reading'] < 0, 'meter_reading'] = 0<save_to_csv>
cat_vec = CountVectorizer(stop_words=[stopwords, string.punctuation], max_features=int(len(cat_counts)*0.5))
cat_matrix = cat_vec.fit_transform(df.category_name)
 cat_matrix_sub = cat_vec.transform(df_sub.category_name )
fold_1.to_csv('submission_fold_bland.csv',index=False )<define_variables>
cat_tokens = list(zip(cat_vec.get_feature_names() , np.array(cat_matrix.sum(axis=0)).ravel()))
INPUT_PATH = Path(".. /input/" )<load_from_csv>
brand_vec = CountVectorizer(stop_words=[stopwords, string.punctuation], max_features=int(len(brand_counts)*0.5))
brand_matrix = brand_vec.fit_transform(df.brand_name)
 brand_matrix_sub = brand_vec.transform(df_sub.brand_name )
train = pd.read_csv(INPUT_PATH / "train.csv")
 test = pd.read_csv(INPUT_PATH / "test.csv")
 train.shape, test.shape<data_type_conversions>
brand_tokens = list(zip(brand_vec.get_feature_names() , np.array(brand_matrix.sum(axis=0)).ravel()))
drop_cols = ["id"]
target_col = "target"
target = train[target_col]
feat_cols = [col for col in train.columns if col not in drop_cols + [target_col]]
train[target_col] = train[target_col].str.replace('Class_', '')
 train[target_col] = train[target_col].astype(int)- 1
target = train[target_col]<init_hyperparams>
name_vec = TfidfVectorizer(min_df=15, stop_words=[stopwords, string.punctuation])
 name_matrix = name_vec.fit_transform(df.name)
 name_matrix_sub = name_vec.transform(df_sub.name )
NFOLDS = 2
RANDOM_STATE = 871972
excluded_column = ['target', 'id']
cols = [col for col in train.columns if col not in excluded_column]
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE)
 params = {"objective": "multi:softprob",
"eval_metric":"mlogloss",
"num_class": 9
}<prepare_x_and_y>
print('Kept %d words.' % len(name_vec.get_feature_names()))
y_pred_test = np.zeros(( len(test), 9))
oof = np.zeros(( len(train), 9))
score = 0
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y=target)) :
print('Fold', fold_n)
 X_train, X_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = target.loc[train_index].astype(int), target.loc[valid_index].astype(int)
 train_data = xgb.DMatrix(X_train[cols], label=y_train)
 valid_data = xgb.DMatrix(X_valid[cols], label=y_valid)
 model = xgb.train(params,
train_data,num_boost_round=30000,
evals=[(valid_data, "valid")],
verbose_eval=4,
maximize=False,
early_stopping_rounds=300)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid[cols]))
oof[valid_index] = y_pred_valid
score += log_loss(y_valid, y_pred_valid)
 y_pred_test += model.predict(xgb.DMatrix(test[cols])) /NFOLDS
print('valid logloss average:', score/NFOLDS, log_loss(target, oof))<save_to_csv>
desc_vec = TfidfVectorizer(max_features=100000,
stop_words=[stopwords, string.punctuation])
 desc_matrix = desc_vec.fit_transform(df.item_description)
 desc_matrix_sub= desc_vec.transform(df_sub.item_description )
sample_submit = pd.read_csv(".. /input/sampleSubmission.csv")
 submit = pd.concat([sample_submit[['id']], pd.DataFrame(y_pred_test)], axis = 1)
 submit.columns = sample_submit.columns
submit.to_csv('submit_xgboost.csv', index=False )<save_model>
cond_matrix = sparse.csr_matrix(pd.get_dummies(df.item_condition_id, sparse=True, drop_first=True))
cond_matrix_sub = sparse.csr_matrix(pd.get_dummies(df_sub.item_condition_id, sparse=True, drop_first=True))
np.save("xgb_oof.npy", oof )<set_options>
ship_matrix = sparse.csr_matrix(df.shipping ).transpose()
ship_matrix_sub = sparse.csr_matrix(df_sub.shipping ).transpose()
%matplotlib inline 
<load_from_csv>
sparse_matrix = sparse.hstack([cat_matrix, brand_matrix, name_matrix, desc_matrix,
cond_matrix, ship_matrix])
 sparse_matrix_sub = sparse.hstack([cat_matrix_sub, brand_matrix_sub, name_matrix_sub,
desc_matrix_sub, cond_matrix_sub, ship_matrix_sub] )
train_base = pd.read_csv('.. /input/otto-group-product-classification-challenge/train.csv')
 test_base = pd.read_csv('.. /input/otto-group-product-classification-challenge/test.csv')
 sample_submit = pd.read_csv('.. /input/otto-group-product-classification-challenge/sampleSubmission.csv' )<load_from_csv>
if sparse_matrix.shape[1] == sparse_matrix_sub.shape[1]:
print('Features check out.')
 else:
print("The number of features in training and test set don't match." )
oof_lgb = pd.read_csv(".. /input/otto-simple-lgb/oof_lgb.csv")
 oof_nn = pd.read_csv(".. /input/otto-simple-nn/oof_nn.csv")
 oof_knn = pd.read_csv(".. /input/otto-knn/oof_knn.csv")
 test_lgb = pd.read_csv(".. /input/otto-simple-lgb/submit_lgb.csv")
 test_nn = pd.read_csv(".. /input/otto-simple-nn/submit_nn.csv")
 test_knn = pd.read_csv(".. /input/otto-knn/submit_knn.csv" )<concatenate>
del(cat_matrix, brand_matrix, name_matrix, desc_matrix, cond_matrix, ship_matrix)
 del(cat_matrix_sub, brand_matrix_sub, name_matrix_sub, desc_matrix_sub, cond_matrix_sub, ship_matrix_sub)
 del(df, df_sub)
 gc.collect()
train = pd.concat([oof_lgb, oof_nn, oof_knn, train_base], axis=1)
 test = pd.concat([test_lgb, test_nn, test_knn, test_base], axis=1 )<data_type_conversions>
def rmsle(pred, true):
assert len(pred)== len(true)
 return np.sqrt(np.mean(np.power(np.log1p(pred)-np.log1p(true), 2)) )
train['target'] = train['target'].str.replace('Class_', '')
 train['target'] = train['target'].astype(int)- 1<choose_model_class>
y_target = np.log10(np.array(y_target)+ 1 )
NFOLDS = 5
RANDOM_STATE = 871975
excluded_column = ['target', 'id']
cols = [c for c in train.columns if c not in(excluded_column + [])]
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE)
<init_hyperparams>
X_train, X_test, y_train, y_test = train_test_split(sparse_matrix, y_target, test_size = 0.1 )
params = {
'bagging_freq': 5, 'bagging_fraction': 0.8, 'boost_from_average':'false', 'boost': 'gbdt',
'feature_fraction': 0.3, 'learning_rate': 0.05, 'max_depth': -1, 'metrics':'multi_logloss',
'min_data_in_leaf': 10, 'min_sum_hessian_in_leaf': 8.0,'num_leaves': 8, 'num_threads': 8,
'tree_learner': 'serial', 'objective': 'multiclass', 'num_class': 9, 'verbosity': 1
}<prepare_x_and_y>
start = time.time()
est_ridge = Ridge(solver='sag', alpha=5)
 est_ridge.fit(X_train, y_train)
 pred_ridge_5 = est_ridge.predict(X_test)
 print(rmsle(10 ** pred_ridge_5 - 1, 10 ** y_test - 1))
print_time(start )
<save_to_csv><EOS> 
pred_sub = est_ridge.predict(sparse_matrix_sub)
 ridge_submission = submission.copy()
ridge_submission['price'] = pd.DataFrame(10 ** pred_sub - 1)
 ridge_submission.to_csv('ridge_test_2.csv', index=False )