Kaggle competition: Mercari Price Suggestion Challenge
submit = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/test.csv')
 X_submit = submit.drop(columns=['id'])
 predictions = model.predict(X_submit)
 pred = pd.DataFrame(predictions, columns=['Class_1', 'Class_2', 'Class_3', 'Class_4', 'Class_5', 'Class_6', 'Class_7', 'Class_8', 'Class_9'])
 result = pd.concat([submit[['id']], pred], axis=1)
 result.to_csv('/kaggle/working/output.csv', index=False)
 print(result.head())
<load_from_csv>
def transform_values(data):
data.fillna(value='missing', inplace = True)
 return data
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0)
 y_reg = train['totals.transactionRevenue']
train.shape, test.shape<prepare_x_and_y>
df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df_train = transform_values(df_train)
 df_test = transform_values(df_test )
y_reg = train['totals.transactionRevenue'].fillna(0 )<data_type_conversions>
def transform_category_name(category_name):
try:
main, sub1, sub2= category_name.split('/')
 return main, sub1, sub2
except:
return 'missing', 'missing', 'missing'
df_train['category_main'], df_train['category_sub1'], df_train['category_sub2'] = zip(*df_train['category_name'].apply(transform_category_name))
df_train[df_train.price != 0.0]
df_train[df_train.item_description != 'missing']
df_test['category_main'], df_test['category_sub1'], df_test['category_sub2'] = zip(*df_test['category_name'].apply(transform_category_name))
df_test[df_test.item_description != 'missing']
train['target'] = y_reg
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
y_reg = train['target']
del train['target']<categorify>
cv = CountVectorizer(ngram_range=(1,2), min_df=7, stop_words='english')
 X_name_train = cv.fit_transform(df_train['name'])
 X_name_test = cv.transform(df_test['name'])
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
cat_cv1 = CountVectorizer()
cat_cv2 = CountVectorizer()
cat_cv3 = CountVectorizer()
X_catmain_train = cat_cv1.fit_transform(df_train['category_main'])
 X_catsub1_train = cat_cv2.fit_transform(df_train['category_sub1'])
 X_catsub2_train = cat_cv3.fit_transform(df_train['category_sub2'])
 X_catmain_test = cat_cv1.transform(df_test['category_main'])
 X_catsub1_test = cat_cv2.transform(df_test['category_sub1'])
 X_catsub2_test = cat_cv3.transform(df_test['category_sub2'] )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<define_variables>
vectorizer = TfidfVectorizer(ngram_range=(1,2), max_features=90000, strip_accents="ascii",
stop_words='english')
 X_description_train = vectorizer.fit_transform(df_train['item_description'])
 X_description_test = vectorizer.transform(df_test['item_description'] )
excluded_features = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime']
categorical_features = [ _f for _f in train.columns\
if(_f not in excluded_features)&(train[_f].dtype == 'object')]
dropcolumns = [c for c in categorical_features if train[c].nunique() == 1]
train = train[[_f for _f in train.columns if _f not in dropcolumns]]
test = test[[_f for _f in test.columns if _f not in dropcolumns]]
categorical_features = [ _f for _f in train.columns\
if(_f not in excluded_features)&(train[_f].dtype == 'object')]
def process_device2(train,test):
print("process device2...")
 train_objs_num = len(train)
 dataset = pd.concat(objs=[train, test], axis=0)
 for f in categorical_features:
dataset[f], indexer = pd.factorize(dataset[f])
 train = dataset[:train_objs_num]
test = dataset[train_objs_num:]
del dataset
return train, test
train, test = process_device2(train,test)
 gc.collect()<load_from_csv>
lb = LabelBinarizer()
X_brand_train = lb.fit_transform(df_train['brand_name'])
 X_brand_test = lb.transform(df_test['brand_name'] )
print('load')
 temp_train = pd.read_csv('.. /input/try-so-hard/temp_train_predictions.csv',
dtype={'fullVisitorId': str}, nrows=None)
 temp_test = pd.read_csv('.. /input/try-so-hard/temp_test_predictions.csv',
dtype={'fullVisitorId': str}, nrows=None)
 print('finished')
 y_reg = train['totals.transactionRevenue']
excluded_features = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime','predictions']
train_features = [_f for _f in train.columns if _f not in excluded_features]<groupby>
X_shipping_train = pd.get_dummies(df_train['shipping'])
 X_condition_train = pd.get_dummies(df_train['item_condition_id'])
 X_shipping_test = pd.get_dummies(df_test['shipping'])
 X_condition_test = pd.get_dummies(df_test['item_condition_id'] )
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_data.sort_index(inplace=True)
 trn_pred_list = temp_train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 trn_pred_list.sort_index(inplace=True )<feature_engineering>
vectorized_data_train = hstack(( X_name_train, X_catmain_train, X_catsub1_train, X_catsub2_train,
X_description_train, X_shipping_train, X_condition_train))
vectorized_data_test = hstack(( X_name_test, X_catmain_test, X_catsub1_test, X_catsub2_test,
X_description_test, X_shipping_test, X_condition_test))
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions, temp_train
gc.collect()
full_data.shape<groupby>
Y_train = np.log1p(df_train['price'])
 clf = Ridge(alpha=5.3, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=None)
 clf.fit(vectorized_data_train, Y_train)
 Y_pred_test = clf.predict(vectorized_data_test )
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_data.sort_index(inplace=True)
 sub_pred_list = temp_test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 sub_pred_list.sort_index(inplace=True )<prepare_output>
submission = pd.DataFrame(data=df_test[['test_id']])
 submission['price'] = np.expm1(Y_pred_test)
 submission.to_csv("submission_ridge.csv", index=False )
<groupby><EOS> 
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5