Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<categorify>
import pandas as pd
import numpy as np
import scipy
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import LabelBinarizer
import gc
trans_train = transforms.Compose([trainsforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([trainsforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)
 optimizer = optim.Adam(params_to_update, lr=0.0001)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.25)
 for epoch in range(6,10):
at = time.time()
scheduler.step()
model.train()
train_acc = []
train_loss = []
for i, data in enumerate(loader_train):
if i%10 == 0:
print('.',end='')
 inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 loss.backward()
optimizer.step()
a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
train_acc.append(roc_auc_score(a, b))
train_loss.append(loss.item())
 model.eval()
valid_acc = []
val_loss = []
for _, data in enumerate(loader_valid):
inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
valid_acc.append(roc_auc_score(a, b))
val_loss.append(loss.item())
 print()
bt = time.time()
print('[Epoch %d] train loss %.6f train acc %.6f valid loss %.6f valid acc %.6f time %.6f' %(
 epoch, np.mean(train_loss), np.mean(train_acc), np.mean(val_loss), np.mean(valid_acc),bt-at))
valid_acc = np.mean(valid_acc)
 if valid_acc > best_val_acc:
best_val_acc = valid_acc
best_epoch = epoch
epoch_since_best = 0
print('save model...')
 torch.save(model.state_dict() , 'model.pth')
 print('saved.')
 else:
epoch_since_best += 1
if epoch_since_best > early_stopping:
break
print('Finished Training')
 print('best_epoch: %d, best_val_acc %.6f' %(best_epoch, best_val_acc))<save_to_csv>
data = {
'train': pd.read_csv(".. /input/train.tsv", sep='\t'),
'test': pd.read_csv(".. /input/test.tsv", sep='\t'),
}
dataset_valid = MyDataset(df_data=sub, data_dir=test_path, transform=trans_valid)
 loader_test = DataLoader(dataset = dataset_valid, batch_size=32, shuffle=False, num_workers=0)
 model.eval()
preds = []
for batch_i,(data, target)in tqdm(enumerate(loader_test)) :
data, target = data.cuda() , target.cuda()
output = model(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 sub['label'] = preds
sub.to_csv('sub.csv', index=False )<set_options>
y_train = np.log1p(data['train']['price'])
 X_train, X_valid, y_train, y_valid = train_test_split(data['train'], y_train, test_size=0.2, random_state=42 )
warnings.filterwarnings('ignore')
 sns.set(style='whitegrid' , palette='tab10')
 %matplotlib inline<load_from_csv>
n_train = X_train.shape[0]
n_valid = X_valid.shape[0]
n_test = data['test'].shape[0]
full_data = pd.concat([X_train, X_valid, data['test']], axis=0)
 del data['train']
gc.collect()
train_df = pd.read_csv('.. /input/train.csv',header = 0)
 test_df = pd.read_csv('.. /input/test.csv',header=0 )<concatenate>
def split_cat(text):
try:
return text.split("/")
 except:
return("Unknown", "Unknown", "Unknown" )
Bike_data=pd.concat([train_WithoutOutliers,test_df],ignore_index=True)
 Bike_data.shape<data_type_conversions>
full_data['general_cat'], full_data['subcat_1'], full_data['subcat_2'] = zip(*full_data['category_name'].apply(lambda x: split_cat(x)))
 full_data["brand_name"] = full_data["brand_name"].fillna("unknown")
 full_data["item_description"] = full_data["item_description"].fillna("No description yet" )
Bike_data['date']=Bike_data.datetime.apply(lambda c : c.split() [0])
 Bike_data['hour']=Bike_data.datetime.apply(lambda c : c.split() [1].split(':')[0] ).astype('int')
 Bike_data['year']=Bike_data.datetime.apply(lambda c : c.split() [0].split('-')[0] ).astype('int')
 Bike_data['month']=Bike_data.datetime.apply(lambda c : c.split() [0].split('-')[1] ).astype('int')
 Bike_data['weekday']=Bike_data.date.apply(lambda c : datetime.strptime(c,'%Y-%m-%d' ).isoweekday())
 Bike_data.head()<feature_engineering>
print("There are %d General categories." % full_data['general_cat'].nunique() )
Bike_data["windspeed_rfr"]=Bike_data["windspeed"]
dataWind0 = Bike_data[Bike_data["windspeed_rfr"]==0]
dataWindNot0 = Bike_data[Bike_data["windspeed_rfr"]!=0]
rfModel_wind = RandomForestRegressor(n_estimators=1000,random_state=42)
 windColumns = ["season","weather","humidity","month","temp","year","atemp"]
rfModel_wind.fit(dataWindNot0[windColumns], dataWindNot0["windspeed_rfr"])
 wind0Values = rfModel_wind.predict(X= dataWind0[windColumns])
 dataWind0.loc[:,"windspeed_rfr"] = wind0Values
Bike_data = dataWindNot0.append(dataWind0)
 Bike_data.reset_index(inplace=True)
 Bike_data.drop('index',inplace=True,axis=1 )<groupby>
print("There are %d cat1 categories." % full_data['subcat_1'].nunique() )
workingday_df=Bike_data[Bike_data['workingday']==1]
workingday_df = workingday_df.groupby(['hour'], as_index=True ).agg({'casual':'mean',
'registered':'mean',
'count':'mean'})
 nworkingday_df=Bike_data[Bike_data['workingday']==0]
nworkingday_df = nworkingday_df.groupby(['hour'], as_index=True ).agg({'casual':'mean',
'registered':'mean',
'count':'mean'})
 fig, axes = plt.subplots(1, 2,sharey = True)
 workingday_df.plot(figsize=(15,5),title = 'The average number of rentals initiated per hour in the working day',ax=axes[0])
 nworkingday_df.plot(figsize=(15,5),title = 'The average number of rentals initiated per hour in the nonworkdays',ax=axes[1] )<groupby>
print("There are %d cat2 categories." % full_data['subcat_2'].nunique() )
temp_df = Bike_data.groupby(['date','weekday'], as_index=False ).agg({'year':'mean',
'month':'mean',
'temp':'median'})
 temp_df.dropna(axis = 0 , how ='any', inplace = True)
 temp_month = temp_df.groupby(['year','month'], as_index=False ).agg({'weekday':'min',
'temp':'median'})
 temp_df['date']=pd.to_datetime(temp_df['date'])
 temp_month.rename(columns={'weekday':'day'},inplace=True)
 temp_month['date']=pd.to_datetime(temp_month[['year','month','day']])
 fig = plt.figure(figsize=(18,6))
ax = fig.add_subplot(1,1,1)
 plt.plot(temp_df['date'] , temp_df['temp'] , linewidth=1.3 , label='Daily average')
 ax.set_title('Change trend of average temperature per day in two years')
 plt.plot(temp_month['date'] , temp_month['temp'] , marker='o', linewidth=1.3 ,
label='Monthly average')
 ax.legend()<data_type_conversions>
full_data['general_cat'] = full_data['general_cat'].astype('category')
 full_data['subcat_1'] = full_data['subcat_1'].astype('category')
 full_data['subcat_2'] = full_data['subcat_2'].astype('category')
 full_data['brand_name'] = full_data['brand_name'].astype('category')
 full_data['item_condition_id'] = full_data['item_condition_id'].astype('category' )
humidity_df = Bike_data.groupby('date', as_index=False ).agg({'humidity':'mean'})
 humidity_df['date']=pd.to_datetime(humidity_df['date'])
 humidity_df=humidity_df.set_index('date')
 humidity_month = Bike_data.groupby(['year','month'], as_index=False ).agg({'weekday':'min',
'humidity':'mean'})
 humidity_month.rename(columns={'weekday':'day'},inplace=True)
 humidity_month['date']=pd.to_datetime(humidity_month[['year','month','day']])
 fig = plt.figure(figsize=(18,6))
ax = fig.add_subplot(1,1,1)
 plt.plot(humidity_df.index , humidity_df['humidity'] , linewidth=1.3,label='Daily average')
 plt.plot(humidity_month['date'], humidity_month['humidity'] ,marker='o',
linewidth=1.3,label='Monthly average')
 ax.legend()
ax.set_title('Change trend of average humidity per day in two years' )<groupby>
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 def norm_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] if len(x)> 1 and x not in stopwords] )
count_df = Bike_data.groupby(['date','weekday'], as_index=False ).agg({'year':'mean',
'month':'mean',
'casual':'sum',
'registered':'sum',
'count':'sum'})
 count_df.dropna(axis = 0 , how ='any', inplace = True)
 count_month = count_df.groupby(['year','month'], as_index=False ).agg({'weekday':'min',
'casual':'mean',
'registered':'mean',
'count':'mean'})
 count_df['date']=pd.to_datetime(count_df['date'])
 count_month.rename(columns={'weekday':'day'},inplace=True)
 count_month['date']=pd.to_datetime(count_month[['year','month','day']])
 fig = plt.figure(figsize=(18,6))
ax = fig.add_subplot(1,1,1)
 plt.plot(count_df['date'] , count_df['count'] , linewidth=1.3 , label='Daily average')
 ax.set_title('Change trend of average number of rentals initiated per day in two years')
 plt.plot(count_month['date'] , count_month['count'] , marker='o',
linewidth=1.3 , label='Monthly average')
 ax.legend()<groupby>
wb = wordbatch.WordBatch(norm_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29, "norm": None, "tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze= True
X_name = wb.fit_transform(full_data['name'])
 del(wb)
 X_name = X_name[:, np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)]
wb = wordbatch.WordBatch(norm_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.0, 0.5],
"hash_size": 2 ** 29, "norm": "l2", "tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze= True
X_description = wb.fit_transform(full_data['item_description'])
 del(wb)
 X_description = X_description[:, np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)]
day_df=Bike_data.groupby('date' ).agg({'year':'mean','season':'mean',
'casual':'sum', 'registered':'sum'
,'count':'sum','temp':'mean',
'atemp':'mean'})
 season_df = day_df.groupby(['year','season'], as_index=True ).agg({'casual':'mean',
'registered':'mean',
'count':'mean'})
 temp_df = day_df.groupby(['year','season'], as_index=True ).agg({'temp':'mean',
'atemp':'mean'} )<groupby>
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(full_data['brand_name'])
 X_cat = lb.fit_transform(full_data['general_cat'])
 X_subcat1 = lb.fit_transform(full_data['subcat_1'])
 X_subcat2 = lb.fit_transform(full_data['subcat_2'])
 X_dummies = csr_matrix(pd.get_dummies(full_data[['item_condition_id', 'shipping']],
sparse=True ).values)
 print(print(X_dummies.shape, X_description.shape, X_brand.shape, X_cat.shape,
X_subcat1.shape, X_subcat2.shape,X_name.shape))
X = hstack(( X_dummies, X_name, X_brand, X_cat, X_subcat1, X_subcat2, X_description)).tocsr()
count_weather = Bike_data.groupby('weather')
 count_weather[['casual','registered','count']].count()<filter>
ridge_model = Ridge(solver='auto', fit_intercept=True, alpha=0.4,
max_iter=200, normalize=False, tol=0.01, random_state = 42 )
Bike_data[Bike_data['weather']==4]<data_type_conversions>
ridge_model.fit(X_train, y_train )
windspeed_df = Bike_data.groupby('date', as_index=False ).agg({'windspeed_rfr':'mean'})
 windspeed_df['date']=pd.to_datetime(windspeed_df['date'])
 windspeed_df=windspeed_df.set_index('date')
 windspeed_month = Bike_data.groupby(['year','month'], as_index=False ).agg({'weekday':'min',
'windspeed_rfr':'mean'})
 windspeed_month.rename(columns={'weekday':'day'},inplace=True)
 windspeed_month['date']=pd.to_datetime(windspeed_month[['year','month','day']])
 fig = plt.figure(figsize=(18,6))
ax = fig.add_subplot(1,1,1)
 plt.plot(windspeed_df.index , windspeed_df['windspeed_rfr'] , linewidth=1.3,label='Daily average')
 plt.plot(windspeed_month['date'], windspeed_month['windspeed_rfr'] ,
marker='o', linewidth=1.3,label='Monthly average')
 ax.legend()
ax.set_title('Change trend of average number of windspeed per day in two years' )<filter>

 def rmsle(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)) )
df2=Bike_data[Bike_data['windspeed']>40]
df2=df2[df2['count']>400]
df2<groupby>
y_valid_pred = ridge_model.predict(X_valid)
 print("RMSL error on valid set:", rmsle(y_valid, y_valid_pred))
print("MAE on valid set:", mean_absolute_error(y_valid, y_valid_pred))
day_df = Bike_data.groupby(['date'], as_index=False ).agg({'casual':'sum','registered':'sum',
'count':'sum', 'workingday':'mean',
'weekday':'mean','holiday':'mean',
'year':'mean'})
 day_df.head()<groupby>
ridge_preds = ridge_model.predict(X_test )
holiday_coun=day_df.groupby('year', as_index=True ).agg({'holiday':'sum'})
 holiday_coun<categorify>
ftrl_model = FTRL(alpha=0.01, beta=0.1, L1=0.00001, L2=1.0, D=X_train.shape[1],
iters=60, inv_link="identity", threads=1)
 ftrl_model.fit(X_train, y_train.reshape(-1))
dummies_month = pd.get_dummies(Bike_data['month'], prefix= 'month')
 dummies_season=pd.get_dummies(Bike_data['season'],prefix='season')
 dummies_weather=pd.get_dummies(Bike_data['weather'],prefix='weather')
 dummies_year=pd.get_dummies(Bike_data['year'],prefix='year')
 Bike_data=pd.concat([Bike_data,dummies_month,dummies_season,dummies_weather,dummies_year],axis=1 )<sort_values>
y_valid_pred = ftrl_model.predict(X_valid)
 y_valid_pred = y_valid_pred.reshape(-1, 1)
 print("RMSL error on valid set:", rmsle(y_valid, y_valid_pred))
print("MAE on valid set:", mean_absolute_error(y_valid, y_valid_pred))
dataTrain = Bike_data[pd.notnull(Bike_data['count'])]
dataTest= Bike_data[~pd.notnull(Bike_data['count'])].sort_values(by=['datetime'])
 datetimecol = dataTest['datetime']
yLabels=dataTrain['count']
yLabels_log=np.log(yLabels)
<drop_column>
ftrl_preds = ftrl_model.predict(X_test )
dropFeatures = ['casual' , 'count' , 'datetime' , 'date' , 'registered' ,
'windspeed' , 'atemp' , 'month','season','weather', 'year' ]
dataTrain = dataTrain.drop(dropFeatures , axis=1)
 dataTest = dataTest.drop(dropFeatures , axis=1 )<predict_on_test>
fm_ftrl_model = FM_FTRL(alpha=0.01, beta=0.1, L1=0.00001, L2=0.1, D=X_train.shape[1], alpha_fm=0.01,
L2_fm=0.0, init_fm=0.01, D_fm=200, e_noise=0.0001, iters=18, inv_link="identity", threads=4)
 fm_ftrl_model.fit(X_train, y_train.reshape(-1))
rfModel = RandomForestRegressor(n_estimators=1000 , random_state = 42)
 rfModel.fit(dataTrain , yLabels_log)
 preds = rfModel.predict(X = dataTrain )<save_to_csv>
y_valid_pred = fm_ftrl_model.predict(X_valid)
 y_valid_pred = y_valid_pred.reshape(-1, 1)
 print("RMSL error on valid set:", rmsle(y_valid, y_valid_pred))
print("MAE on valid set:", mean_absolute_error(y_valid, y_valid_pred))
predsTest= rfModel.predict(X = dataTest)
 submission=pd.DataFrame({'datetime':datetimecol , 'count':[max(0,x)for x in np.exp(predsTest)]})
 submission.to_csv('bicycle_predictions.csv',index=False )<import_modules>
ft_ftrl_preds = fm_ftrl_model.predict(X_test )
import pandas as pd<import_modules>
lgb_label = y_train.ravel()
lgb_y_valid = y_valid.ravel()
lgb_train = lgb.Dataset(X_train, label=lgb_label)
 lgb_eval = lgb.Dataset(X_valid, lgb_y_valid, reference=lgb_train)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'l2', 'rmse'},
'learning_rate': 0.6,
'feature_fraction': 0.65,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'verbose': 1,
'data_random_seed': 42,
'max_depth': 3,
'nthread': 4,
'min_data_in_leaf': 100,
'max_bin': 31
}
print('Start training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=5500,
valid_sets=lgb_eval,
early_stopping_rounds=1000 )
import pandas as pd<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 train.head()<load_from_csv>
test = pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()<data_type_conversions>
<feature_engineering><EOS> 
lgbm_preds = gbm.predict(X_test, num_iteration=gbm.best_iteration)
 lgbm_preds = lgbm_preds.reshape(-1, 1)
 ftrl_preds = ftrl_preds.reshape(-1, 1)
 ft_ftrl_preds = ft_ftrl_preds.reshape(-1, 1)
 preds = ridge_preds*0.05 + lgbm_preds*0.06 + ftrl_preds*0.25 + ft_ftrl_preds*0.64
data['test']["price"] = np.expm1(preds)
 data['test'][["test_id", "price"]].to_csv("submission_ridge_FT_FTRL_LGBM.csv", index = False )