Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<save_to_csv>
import pandas as pd
import numpy as np
import scipy
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from sklearn.linear_model import Ridge
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from scipy.optimize import minimize
import matplotlib.pyplot as plt
import gc
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<set_options>
print("Reading in Data")
 df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test_stg2.tsv', sep='\t')
 nrow_train=df_train.shape[0]
nrow_test=df_test.shape[0]
Y_train=np.log1p(df_train["price"])
 print('Size of train set',nrow_train)
 print('Size of test set',nrow_test)
 df=pd.concat([df_train,df_test],axis=0,sort=True)
 df_test=pd.DataFrame(df_test["test_id"])
 NUM_BRAND=2500
df["category_name"]=df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"]=df["brand_name"].fillna("Unknown")
 pop_brands=df["brand_name"].value_counts().index[1:NUM_BRAND+1]
df.loc[~df["brand_name"].isin(pop_brands),"brand_name"]="Other"
df["brand_name"]=df["brand_name"].astype("category")
 df["item_description"]=df["item_description"].fillna("None")
 df["item_condition_id"]=df["item_condition_id"].astype("category")
 df_train.head()
torch.cuda.is_available()<set_options>
del df_train
gc.collect
print("Is a GPU available? {}".format(is_available()))
print("The name of the device is {}".format(get_device_name()))
print("Device capability is {}".format(get_device_capability(0)))
 print("Available GPU memory is {0:.{1}f} GB ".format(get_device_properties(0 ).total_memory /(1024 ** 3), 1))<load_from_csv>
NAME_MIN_DF=10
MAX_FEAT_DESCP=50000
print("Encodings")
 print("Condition Encoders")
 vect_condition=LabelBinarizer(sparse_output=True)
 X_Condition=vect_condition.fit_transform(df["item_condition_id"])
 print("Shipping Encoders")
 vect_shipping=LabelBinarizer(sparse_output=True)
 X_Shipping=vect_shipping.fit_transform(df["shipping"])
 print("Name Encoders")
 count_name=CountVectorizer(min_df=NAME_MIN_DF)
 X_name=count_name.fit_transform(df["name"])
 print("Category Encoders")
 count_category=CountVectorizer()
X_category=count_category.fit_transform(df["category_name"])
 print("Brand Encoders")
 count_brand=CountVectorizer()
X_brand=count_brand.fit_transform(df["brand_name"])
 print("Descp Encoders")
 count_descp=TfidfVectorizer(max_features=MAX_FEAT_DESCP,
ngram_range=(1,3),
stop_words="english")
 X_descp=count_descp.fit_transform(df["item_description"])
df = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 os.makedirs("images")
 def bbox(image):

 HEIGHT = image.shape[0]
WIDTH = image.shape[1]
for i in range(image.shape[1]):
if(image[:, i] > 0 ).sum() >= 1:
x_min = i - 1 if(i > 1)else 0
break
for i in reversed(range(image.shape[1])) :
if(image[:, i] > 0 ).sum() >= 1:
x_max = i + 2 if(i < WIDTH - 2)else WIDTH
break
for i in range(image.shape[0]):
if(image[i] > 0 ).sum() >= 1:
y_min = i - 1 if(i > 1)else 0
break
for i in reversed(range(image.shape[0])) :
if(image[i] > 0 ).sum() >= 1:
y_max = i + 2 if(i < HEIGHT - 2)else HEIGHT
break
return x_min, y_min, x_max, y_max
ids = []
labels = []
for i,row in df.iterrows() :
id = "{}".format(i)
 img = np.reshape(row[1:].values, newshape=(28,28)).astype(np.uint8)
 x_min, y_min, x_max, y_max = bbox(img)
 img = img[y_min:y_max, x_min:x_max]
x_min, y_min, x_max, y_max = bbox(img)
 img = img[y_min:y_max, x_min:x_max]
img = np.pad(img,(2,2), "constant", constant_values=(0,0))
img[img >= 50] = 255
img[img < 50] = 0
img = Image.fromarray(img)
 img = img.resize(( 28,28), Image.ANTIALIAS)
 label = 0
img.save("images/{}.png".format(id))
ids.append(id)
 labels.append(label)
 label_df = pd.DataFrame({"id": ids, "label": labels})
<train_model>
del df
gc.collect
class KannadaMNIST(object):
def __init__(self
, image_folder : str
, labels : pd.DataFrame
, transforms : list = None 
):

 self.image_folder = image_folder
self.labels = labels
self.transforms = transforms
self.index = np.arange(self.labels.shape[0])
 def __len__(self):
return len(self.index)
 def __getitem__(self, idx):
label = self.labels.iloc[self.index[idx]]
img = Image.open(self.image_folder + "/" + label["id"] + ".png")
 img_ = self.transforms(img)
 return {"image": img_, "id":int(label["id"])}<choose_model_class>
X=scipy.sparse.hstack([X_Shipping,X_Condition,X_brand,
X_category,X_descp,X_name] ).tocsr()
def conv1d(ni, no, ks=1, stride=1, padding=0, bias=False):
	"Create and initialize a `nn.Conv1d` layer with spectral normalization."
	conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias)
 	nn.init.kaiming_normal_(conv.weight)
 	if bias: conv.bias.data.zero_()
	return nn.utils.spectral_norm(conv)
 class SimpleSelfAttention(nn.Module):
	
	def __init__(self, n_in, ks=1, sym=False):
		super().__init__()
		self.conv = conv1d(n_in, n_in, ks, padding=ks//2, bias=False)
 		self.gamma = nn.Parameter(torch.Tensor([0.]))
		self.sym = sym
		self.n_in = n_in
		
	def forward(self, x):
		if self.sym:
			
			c = self.conv.weight.view(self.n_in,self.n_in)
 			c =(c + c.t())/2
			self.conv.weight = c.view(self.n_in,self.n_in,1)
 				
		size = x.size()
		x = x.view(*size[:2],-1)
 		
		
		
		
		convx = self.conv(x)
 		xxT = torch.bmm(x, x.permute(0,2,1 ).contiguous())
 		o = torch.bmm(xxT, convx)
 		o = self.gamma * o + x
		
		return o.view(*size ).contiguous()<import_modules>
del X_descp
del X_brand
del X_category
del X_name
del X_Shipping
del X_Condition
gc.collect
class KannadaNet(nn.Module):
def __init__(self
, input_shape
, n_class 
):

 super(KannadaNet, self ).__init__()
self._build_layers(input_shape=input_shape, n_class=n_class)
 self._init_weight()
print("Model contains {} parameters".format(sum(p.numel() for p in self.parameters() if p.requires_grad)))
 def forward(self, x):
x = self._conv_forward(x)
 x = self.fc_1(x)
 x = F.relu(x)
 x = self.bn_1(x)
 x = self.fc_2(x)
 x = F.relu(x)
 x = self.bn_2(x)
 confidence = self.confidence(x)
 out = self.classifier(x)
 return {"prediction":out.cpu().float() , "confidence":confidence.cpu().float() }
def _build_layers(self, input_shape, n_class):

 self.stem = Conv2D_BN(in_channels =input_shape[0]
, out_channels=32
, padding =1 
)
 self.inception_res_1 = InceptionBlock(in_shape=[32,28,28], residual=True)
 self.attention_1 = SimpleSelfAttention(self.inception_res_1.output_shape[0])
 self.inception_res_2 = InceptionBlock(in_shape=[int(x / y)for x, y in zip(self.inception_res_1.output_shape, [1,2,2])], residual=True)
 self.attention_2 = SimpleSelfAttention(self.inception_res_2.output_shape[0])
 self.inception_res_3 = InceptionBlock(in_shape=[int(x / y)for x, y in zip(self.inception_res_2.output_shape, [1, 2, 2])], residual=True)
 self.attention_3 = SimpleSelfAttention(self.inception_res_3.output_shape[0])
 self.pool = nn.MaxPool2d(kernel_size=2)
 conv_out_dims = self._conv_forward(torch.zeros([1] + input_shape, dtype=torch.float32))
self.fc_1 = nn.Linear(in_features=conv_out_dims.shape[1], out_features=512)
 self.bn_1 = nn.BatchNorm1d(num_features=512)
 self.fc_2 = nn.Linear(in_features=512, out_features=512)
 self.bn_2 = nn.BatchNorm1d(num_features=512)
 self.confidence = nn.Linear(in_features=512, out_features=1)
 self.classifier = nn.Linear(in_features=512, out_features=n_class)
 def _conv_forward(self, x):

 x = self.stem(x)
 x = self.inception_res_1(x)
 x = self.attention_1(x)
 x = self.pool(x)
 x = self.inception_res_2(x)
 x = self.attention_2(x)
 x = self.pool(x)
 x = self.inception_res_3(x)
 x = self.attention_3(x)
 x = self.pool(x)
 x = torch.flatten(x, start_dim=1)
 return x
def _init_weight(self):
for m in self.modules() :
print(m)
 if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
class InceptionBlock(nn.Module):
def __init__(self, in_shape, residual=True):
super(InceptionBlock, self ).__init__()
self.residual = residual
self.conv_1x1_1 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_1 = nn.BatchNorm2d(num_features=self.conv_1x1_1.out_channels)
 self.conv_1x1_2 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]))
self.bn_2 = nn.BatchNorm2d(num_features=self.conv_1x1_2.out_channels)
 self.conv_1x1_3 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_3 = nn.BatchNorm2d(num_features=self.conv_1x1_3.out_channels)
 self.conv_1x1_4 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_4 = nn.BatchNorm2d(num_features=self.conv_1x1_4.out_channels)
 self.conv_3x3_1 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_2.out_channels, out_channels=self.conv_1x1_1.out_channels * 2, padding=1)
 self.bn_5 = nn.BatchNorm2d(num_features=self.conv_3x3_1.out_channels)
 self.conv_3x3_2 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_3.out_channels, out_channels=int(in_shape[0]/4), padding=1)
 self.bn_6 = nn.BatchNorm2d(num_features=self.conv_3x3_2.out_channels)
 self.conv_3x3_3 = nn.Conv2d(kernel_size=3, in_channels=self.conv_3x3_2.out_channels, out_channels=int(in_shape[0]/4), padding=1)
 self.bn_7 = nn.BatchNorm2d(num_features=self.conv_3x3_3.out_channels)
 self.pool_3x3 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
 self.output_shape = self._get_out_shape(in_shape)
 def forward(self, x):
x_1 = self.conv_1x1_1(x)
 x_1 = F.relu(x_1)
 x_1 = self.bn_1(x_1)
 x_2 = self.conv_1x1_2(x)
 x_2 = F.relu(x_2)
 x_2 = self.bn_2(x_2)
 x_2 = self.conv_3x3_1(x_2)
 x_2 = F.relu(x_2)
 x_2 = self.bn_5(x_2)
 x_3 = self.conv_1x1_3(x)
 x_3 = F.relu(x_3)
 x_3 = self.bn_3(x_3)
 x_3 = self.conv_3x3_2(x_3)
 x_3 = F.relu(x_3)
 x_3 = self.bn_6(x_3)
 x_3 = self.conv_3x3_3(x_3)
 x_3 = F.relu(x_3)
 x_3 = self.bn_7(x_3)
 x_4 = self.pool_3x3(x)
 x_4 = self.conv_1x1_4(x_4)
 x_4 = F.relu(x_4)
 x_4 = self.bn_4(x_4)
 if self.residual:
x = torch.cat([x_1, x_2, x_3, x_4, x], dim=1)
 else:
x = torch.cat([x_1, x_2, x_3, x_4], dim=1)
 return x
def _get_out_shape(self, input_shape):
return list(self.forward(torch.zeros([1] + input_shape)).shape[1:])
 class Conv2D_BN(nn.Module):
def __init__(self, in_channels, out_channels, kernel=3, stride=1, padding=1, activation=True):

 super(Conv2D_BN, self ).__init__()
self.activation = activation
self.conv = nn.Conv2d(in_channels =in_channels
, out_channels=out_channels
, kernel_size =kernel
, stride =stride
, padding =padding
, bias =True 
)
 self.bn = nn.BatchNorm2d(num_features=out_channels, momentum=0.9)
 def forward(self, x):
x = self.conv(x)
 if self.activation:
x = F.relu(x)
 x = self.bn(x)
 return x
def block_forward(layers, x):
for layer in layers:
x = layer(x)
 return x
def Mish(x):
r
return x * torch.tanh(F.softplus(x)) 
<load_pretrained>
X_train=X[:nrow_train]
X_test=X[nrow_train:]
dtest=xgb.DMatrix(X_test )
dataset = KannadaMNIST(image_folder="images", labels=label_df, transforms=transforms.ToTensor())
 loader = torch.utils.data.DataLoader(dataset =dataset
, batch_size =256
, shuffle =False
, pin_memory =True
)<categorify>
del X
gc.collect
for sample in loader:
test = sample["image"][0]
break
test = transforms.functional.to_pil_image(test)
 test<set_options>
print("Cross-Validation")
 xgb_pred_val_index = np.zeros(X_train.shape[0])
 ridge_pred_val_index = np.zeros(X_train.shape[0])
 lgb_pred_val_index = np.zeros(X_train.shape[0])
 xgb_pred_all_sum=[]
ridge_pred_all_sum=[]
lgb_pred_all_sum=[]
xgb_cv_RMSLE_sum=0
ridge_cv_RMSLE_sum=0
lgb_cv_RMSLE_sum=0
folds=3
kf = KFold(n_splits=folds, random_state=1001)
 for i,(train_index, val_index)in enumerate(kf.split(X_train, Y_train)) :
x_train, x_val = X_train[train_index], X_train[val_index]
y_train, y_val = Y_train[train_index], Y_train[val_index]
dtrain=xgb.DMatrix(x_train,y_train)
 dval=xgb.DMatrix(x_val)
 deval=xgb.DMatrix(x_val,y_val)
 params = {
'booster': 'gblinear',
'objective': 'reg:linear',
'gamma': 0,
'max_depth': 10,
'lambda': 0,
'subsample': 0.85,
'colsample_bytree': 0.9,
'min_child_weight': 17,
'silent': 1,
'eta': 0.4,
'seed': 1001,
'nthread': 4,
'eval_metric':'rmse'
}
plst = params.items()
evallist = [(deval, 'eval'),(dtrain, 'train')]
num_round=300
model=xgb.train(plst,dtrain,num_round,evallist, verbose_eval=100,early_stopping_rounds=100)
 xgb_pred_val=model.predict(dval)
 xgb_RMSLE=np.sqrt(mean_squared_error(xgb_pred_val,y_val))
print('
Fold %02d XGBoost RMSLE: %.6f' %(( i + 1), xgb_RMSLE))
xgb_pred_all=model.predict(dtest)
 del dtrain
del dval
del deval
gc.collect()
params = {
'boosting': 'gbdt',
'max_depth': 7,
'min_data_in_leaf': 80,
'num_leaves': 30,
'learning_rate': 0.4,
'objective': 'regression',
'metric': 'rmse',
'nthread': 4,
'bagging_freq': 1,
'subsample': 0.9,
'colsample_bytree': 0.7,
'min_child_weight': 17,
'is_unbalance': False,
'verbose': -1,
'seed': 1001,
'max_bin':511,
'num_threads':4
}
dtrain = lgb.Dataset(x_train, label=y_train)
 deval = lgb.Dataset(x_val, label=y_val)
 watchlist = [dtrain, deval]
watchlist_names = ['train', 'val']
model = lgb.train(params,
train_set=dtrain,
num_boost_round=3000,
valid_sets=watchlist,
valid_names=watchlist_names,
early_stopping_rounds=100,
verbose_eval=300)
 lgb_pred_val = model.predict(x_val)
 lgb_RMSLE = np.sqrt(mean_squared_error(lgb_pred_val,y_val))
print(' Fold %02d LightGBM RMSLE: %.6f' %(( i + 1), lgb_RMSLE))
lgb_pred_all = model.predict(X_test)
 del dtrain
del deval
gc.collect()
model=Ridge(solver='sag',alpha=4.75)
 model.fit(x_train,y_train)
 ridge_pred_val=model.predict(x_val)
 ridge_RMSLE=np.sqrt(mean_squared_error(ridge_pred_val,y_val))
print('
Fold %02d Ridge RMSLE: %.6f' %(( i + 1), ridge_RMSLE))
ridge_pred_all=model.predict(X_test)
 del x_train
del y_train
del x_val
del y_val
gc.collect()
xgb_pred_val_index[val_index] = xgb_pred_val
ridge_pred_val_index[val_index] = ridge_pred_val
lgb_pred_val_index[val_index] = lgb_pred_val
if i > 0:
xgb_pred_all_sum = xgb_pred_all_sum + xgb_pred_all
ridge_pred_all_sum = ridge_pred_all_sum + ridge_pred_all
lgb_pred_all_sum = lgb_pred_all_sum + lgb_pred_all
else:
xgb_pred_all_sum = xgb_pred_all
ridge_pred_all_sum = ridge_pred_all
lgb_pred_all_sum = lgb_pred_all
xgb_cv_RMSLE_sum = xgb_cv_RMSLE_sum + xgb_RMSLE
ridge_cv_RMSLE_sum = ridge_cv_RMSLE_sum + ridge_RMSLE
lgb_cv_RMSLE_sum = lgb_cv_RMSLE_sum + lgb_RMSLE
xgb_cv_avg_score=xgb_cv_RMSLE_sum/folds
ridge_cv_avg_score=ridge_cv_RMSLE_sum/folds
lgb_cv_avg_score=lgb_cv_RMSLE_sum/folds
xgb_val_real_RMSLE=np.sqrt(mean_squared_error(xgb_pred_val_index,Y_train))
ridge_val_real_RMSLE=np.sqrt(mean_squared_error(ridge_pred_val_index,Y_train))
lgb_val_real_RMSLE=np.sqrt(mean_squared_error(lgb_pred_val_index,Y_train))
print('
Average XGBoost RMSLE(cv):\t%.6f' % xgb_cv_avg_score)
 print(' Out-of-fold XGBoost RMSLE:\t%.6f' % xgb_val_real_RMSLE)
 print('
Average LightGBM RMSLE(cv):\t%.6f' % lgb_cv_avg_score)
 print(' Out-of-fold LightGBM RMSLE:\t%.6f' % lgb_val_real_RMSLE)
 print('
Average Ridge RMSLE(cv):\t%.6f' % ridge_cv_avg_score)
 print(' Out-of-fold Ridge RMSLE:\t%.6f' % ridge_val_real_RMSLE)
 xgb_pred_all_avg=xgb_pred_all_sum/folds
ridge_pred_all_avg=ridge_pred_all_sum/folds
lgb_pred_all_avg=lgb_pred_all_sum/folds
<save_to_csv><EOS> 
def rmse_min_func(weights):
final_prediction=0
for weight,prediction in zip(weights,blend_train):
final_prediction+=weight*prediction
return np.sqrt(mean_squared_error(Y_train,final_prediction))
blend_train = []
blend_test = []
blend_train.append(xgb_pred_val_index)
 blend_train.append(lgb_pred_val_index)
 blend_train.append(ridge_pred_val_index)
 blend_train=np.array(blend_train)
 blend_test.append(xgb_pred_all_avg)
 blend_test.append(lgb_pred_all_avg)
 blend_test.append(ridge_pred_all_avg)
 blend_test=np.array(blend_test)
 print('
Finding Blending Weights...')
 res_list=[]
weight_list=[]
for k in range(20):
starting_value=np.random.uniform(-1,1,len(blend_train))
bounds=[(-1,1)]*len(blend_train)
 res=minimize(
 rmse_min_func,
starting_value,
method='L-BFGS-B',
bounds=bounds,
options={'disp':False,
'maxiter':100000})
 res_list.append(res['fun'])
 weight_list.append(res['x'])
 print('{iter}\tScore: {score}\tWeights: {weights}'.format(
 iter=(k+1),
score=round(res['fun'],6),
weights='\t'.join([str(round(item,10)) for item in res['x']])))
 bestSC=np.min(res_list)
 bestweight=weight_list[np.argmin(res_list)]
print('
Ensemble Score:{best_score}'.format(best_score=bestSC))
print('
Best Weights:{weight}'.format(weight=bestweight))
test_price=np.zeros(len(blend_test[0]))
train_price =np.zeros(len(blend_train[0]))
print('
Your final model:')
 for k in range(len(blend_test)) :
print('%.6f * model-%d'%(bestweight[k],(k+1)))
 test_price+=blend_test[k]*bestweight[k]
train_price+= blend_train[k] * bestweight[k]
df_test["price"]=np.expm1(test_price)
 print("Generatig File")
 df_test[["test_id","price"]].to_csv("submission.csv",index=False )