Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<load_from_csv>
import numpy as np
import pandas as pd
sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv', index_col='id' )<define_variables>
train = pd.read_csv('.. /input/train.tsv', sep='\t')
 test = pd.read_csv('.. /input/test.tsv', sep='\t')
 train["source"] = "train"
test["source"] = "test"
train = train.rename(columns={"train_id": "id"})
 test = test.rename(columns={"test_id": "id"})
 data = pd.concat([train,test] )
binvar = ['bin_' + str(i)for i in range(1,5)]
ordvar = ['ord_' + str(i)for i in range(6)]
nomvar = ['nom_' + str(i)for i in range(10)]
dmvar = ['day', 'month']<prepare_x_and_y>
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label" )
df_work = df_train.copy()
y_train = df_work['target'].copy()
df_work = pd.concat([df_work.drop('target', axis = 1), df_test] )<drop_column>
data['general_cat'], data['subcat_1'], data['subcat_2'] = \
zip(*data['category_name'].apply(lambda x: split_cat(x)) )
df_work.drop('bin_0', inplace=True, axis=1 )<categorify>
data.drop('category_name',axis=1,inplace=True )
df_work['bin_3'] = df_work['bin_3'].map({'F':0, 'T':1})
 df_work['bin_4'] = df_work['bin_4'].map({'N':0, 'Y':1} )<feature_engineering>
data.brand_name = data.brand_name.fillna("None" )
df_work['ord_0'] = df_work['ord_0'] - 1<categorify>
le = LabelEncoder()
labels = ["brand_name","general_cat","subcat_1","subcat_2"]
for label in labels:
data[label] = data[label].astype(str)
 le.fit(np.hstack(data[label]))
data[label] = le.transform(data[label] )
ord1dict = {'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
df_work['ord_1'] = df_work['ord_1'].map(ord1dict )<categorify>
data.general_cat = data.general_cat.astype(str)
 vectorizer = CountVectorizer(token_pattern='\d+')
 x = vectorizer.fit_transform(data.general_cat )
ord2dict = {'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
df_work['ord_2'] = df_work['ord_2'].map(ord2dict )<categorify>
data.item_description = data.item_description.fillna("None")
 data.item_condition_id = data.item_condition_id.astype(str)
 data.shipping = data.shipping.astype(str)
 data.general_cat = data.general_cat.astype(str)
 data.subcat_1 = data.subcat_1.astype(str)
 data.subcat_2 = data.subcat_2.astype(str)
 data.brand_name = data.brand_name.astype(str )
oe = OrdinalEncoder(categories='auto')
 df_work[ordvar[3:]] = oe.fit_transform(df_work[ordvar[3:]])
 for var, cl in zip(ordvar[3:], oe.categories_):
print(var)
 print(cl )<normalization>
default_preprocessor = CountVectorizer().build_preprocessor()
def preprocessor(field):
field_idx = list(data.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor = preprocessor('name'))), 
('general_cat', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('general_cat'))), 
('subcat_1', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('subcat_1'))), 
('subcat_2', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('subcat_2'))), 
('brand_name', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('item_condition_id'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=preprocessor('item_description'))),
] )
df_work[ordvar] = StandardScaler().fit_transform(df_work[ordvar] )<count_unique_values>
X = vectorizer.fit_transform(data.values )
df_work[nomvar[5:]].nunique()<feature_engineering>
trainData = X[:train.shape[0]]
target = np.log1p(train.price)
 testData = X[train.shape[0]:]
df_work['nom_5'] = df_work['nom_5'].str[4:]
df_work['nom_6'] = df_work['nom_6'].str[3:]
df_work['nom_7'] = df_work['nom_7'].str[3:]
df_work['nom_8'] = df_work['nom_8'].str[3:]
df_work['nom_9'] = df_work['nom_9'].str[3:]<count_unique_values>
X_train, X_valid, y_train, y_valid = train_test_split(trainData, target, test_size=0.3, random_state=0 )
df_work[nomvar[5:]].nunique()<categorify>
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV
enc = OneHotEncoder(categories = 'auto', dtype = 'float64', drop = 'first')
 nom_matrix = enc.fit_transform(df_work[nomvar])
 df_work.drop(nomvar, inplace=True, axis=1 )<categorify>
ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=1.0,
max_iter=100, normalize=False, tol=0.05, random_state = 1, 
)
 ridge_model.fit(X_train, y_train)
 val_preds= ridge_model.predict(X_valid)
 score = mean_squared_error(y_valid, np.array(val_preds))
print("Ridge Score: "+str(np.sqrt(score)) )
enc = OneHotEncoder(categories='auto', dtype = 'float64', drop = 'first')
 dm_matrix = enc.fit_transform(df_work[dmvar])
 df_work.drop(dmvar, inplace=True, axis=1 )<data_type_conversions>
ridge_modelCV = RidgeCV(
 fit_intercept=True, alphas=[5.0],
normalize=False, cv = 2, scoring='neg_mean_squared_error', 
)
 ridge_modelCV.fit(X_train, y_train)
 val_preds= ridge_modelCV.predict(X_valid)
 score = mean_squared_error(y_valid, np.array(val_preds))
print("RidgeCV Score: "+str(np.sqrt(score)) )
df_work_sprs =scipy.sparse.hstack([nom_matrix,
scipy.sparse.coo_matrix(df_work ).astype('float64'),
dm_matrix] ).tocsr()
display(df_work_sprs )<prepare_x_and_y>
preds = ridge_modelCV.predict(testData)
 submission= test[["id"]]
submission["price"] = np.expm1(preds)
 submission.rename(columns={"id": "test_id"})
 submission.to_csv("submission.csv",header=["test_id","price"], index=False )