Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<predict_on_test>
import numpy as np
import pandas as pd
import os
from fastai import *
from fastai.tabular import *
from sklearn.metrics import cohen_kappa_score
test_denoised = test.copy()
test_denoised[cont_features] = autoencoder.predict(test_denoised[cont_features])
 train_denoised = train.copy()
train_denoised[cont_features] = autoencoder.predict(train_denoised[cont_features] )<split>
seed = 42
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
if torch.cuda.is_available() : torch.cuda.manual_seed_all(seed )
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_3 = np.zeros(len(train))
score_list = []
fold = 1
test_preds_3 = []
seed_list = [None,2,3]
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train_denoised, X_val_denoised = train_denoised.iloc[train_index], train_denoised.iloc[test_index]
print(X_train_denoised.shape)
 X_train_denoised = pd.concat([X_train_denoised,X_train],axis=0)
 y_train = pd.concat([y_train,y_train],axis=0)
 print(X_train_denoised.shape)
 y_pred_list = []
for seed in seed_list:
dtrain = lgbm.Dataset(X_train_denoised[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":7,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 dtrain = lgbm.Dataset(X_train_denoised[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 350,
"lambda_l1":7,
"lambda_l2":1,
"learning_rate":0.003,
'min_child_samples': 35,
"bagging_fraction":0.8,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=200,
init_model = model 
)
 y_pred_list.append(model.predict(X_val[cont_features]))
print(np.sqrt(mean_squared_error(y_val, np.mean(y_pred_list,axis=0))))
test_preds_3.append(model.predict(test[cont_features]))
oof_3[test_index] = np.mean(y_pred_list,axis=0)
 score = np.sqrt(mean_squared_error(y_val, oof_3[test_index]))
score_list.append(score)
 print(f"RMSE Fold-{fold} : {score}")
 fold+=1
np.mean(score_list )<compute_test_metric>
train_csv = pd.read_csv('.. /input/train/train.csv', low_memory=False)
 test_csv = pd.read_csv('.. /input/test/test.csv', low_memory=False)
 def preprocess(csv):
csv['Description_len'] = [len(str(tt)) for tt in csv['Description']]
csv['Name_len'] = [len(str(tt)) for tt in csv['Name']]
return csv
train_csv = preprocess(train_csv)
 test_csv = preprocess(test_csv )
print(np.mean(score_list))
score_list<feature_engineering>
cat_names = ['Type','Breed1','Breed2','Gender','Color1','Color2','State','Color3','FurLength', 'Vaccinated','Dewormed','Sterilized','Health']
cont_names = ['Age', 'MaturitySize', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'Description_len', 'Name_len']
train["3_preds"] = oof_3
test["3_preds"] = np.mean(test_preds_3,axis=0 )<save_to_csv>
bs = len(train_csv)
 procs = [FillMissing, Categorify, Normalize]
df = TabularList.from_df(train_csv, path='.. /input', cat_names=cat_names, cont_names=cont_names, procs=procs)\ 
.no_split() \ 
.label_from_df(cols='AdoptionSpeed')
 df_test = TabularList.from_df(test_csv, path='.. /input', cat_names=cat_names, cont_names=cont_names, processor=df.train.x.processor)
 data = df.add_test(df_test ).databunch(num_workers=0, bs=bs )
train[["id","1_preds","2_preds","3_preds"]].to_csv("train_preds.csv",index=False)
 test[["id","1_preds","2_preds","3_preds"]].to_csv("test_preds.csv",index=False )<load_from_csv>
def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Optional[nn.Module]=None):
"Sequence of batchnorm(if `bn`), dropout(with `p`)and linear(`n_in`,`n_out`)layers followed by `actn`."
layers = [nn.BatchNorm1d(n_in, track_running_stats=False)] if bn else []
if p != 0: layers.append(nn.Dropout(p))
layers.append(nn.Linear(n_in, n_out))
if actn is not None: layers.append(actn)
 return layers
class TabularModel(nn.Module):
"Basic model for tabular data."
def __init__(self, emb_szs:ListSizes, n_cont:int, out_sz:int, layers:Collection[int], ps:Collection[float]=None,
emb_drop:float=0., y_range:OptRange=None, use_bn:bool=True, bn_final:bool=False):
super().__init__()
ps = ifnone(ps, [0]*len(layers))
ps = listify(ps, layers)
 self.embeds = nn.ModuleList([embedding(ni, nf)for ni,nf in emb_szs])
 self.emb_drop = nn.Dropout(emb_drop)
 self.bn_cont = nn.BatchNorm1d(n_cont, track_running_stats=False)
 n_emb = sum(e.embedding_dim for e in self.embeds)
 self.n_emb,self.n_cont,self.y_range = n_emb,n_cont,y_range
sizes = self.get_sizes(layers, out_sz)
 actns = [nn.ReLU(inplace=True)] *(len(sizes)-2)+ [None]
layers = []
for i,(n_in,n_out,dp,act)in enumerate(zip(sizes[:-1],sizes[1:],[0.]+ps,actns)) :
layers += bn_drop_lin(n_in, n_out, bn=use_bn and i!=0, p=dp, actn=act)
 if bn_final: layers.append(nn.BatchNorm1d(sizes[-1]))
layers.append(nn.Softmax(dim=1))
self.layers = nn.Sequential(*layers)
 def get_sizes(self, layers, out_sz):
return [self.n_emb + self.n_cont] + layers + [out_sz]
def forward(self, x_cat:Tensor, x_cont:Tensor)-> Tensor:
if self.n_emb != 0:
x = [e(x_cat[:,i])for i,e in enumerate(self.embeds)]
x = torch.cat(x, 1)
 x = self.emb_drop(x)
 if self.n_cont != 0:
x_cont = self.bn_cont(x_cont)
 x = torch.cat([x, x_cont], 1)if self.n_emb != 0 else x_cont
x = self.layers(x)
 if self.y_range is not None:
x =(self.y_range[1]-self.y_range[0])* torch.sigmoid(x)+ self.y_range[0]
return x
train["4_preds"] = pd.read_csv(".. /input/nn-with-categorical-embedding/nn_preds_train.csv")["4_preds"]
test["4_preds"] = pd.read_csv(".. /input/nn-with-categorical-embedding/nn_preds_test.csv")["4_preds"]<compute_test_metric>
unique, counts = np.unique(train_csv['AdoptionSpeed'], return_counts=True)
 E_hist = counts / sum(counts)
 ww = [sum([E_hist[k]*(k-i)**2 for k in range(5)])for i in range(5)]
class KappaLoss(nn.Module):
def forward(self, y1, y2, *args):
numer =(torch.matmul(y2.float().reshape([-1,1]),torch.tensor(()).new_ones(( 1, 5)).cuda())-
torch.matmul(torch.tensor(()).new_ones(( bs, 1)) , torch.tensor(range(5), dtype=torch.float ).reshape([1,5])).cuda())**2
numer =(numer * y1 ).sum()
denom = torch.matmul(torch.tensor(()).new_ones(( bs, 1)) , torch.tensor(ww ).reshape([1,5])).cuda()
denom =(denom * y1 ).sum()
loss = numer / denom
return loss
print("1st model score: ",np.sqrt(mean_squared_error(train["target"], train["1_preds"])))
 print("2nd model score: ",np.sqrt(mean_squared_error(train["target"], train["2_preds"])))
 print("3rd model score: ",np.sqrt(mean_squared_error(train["target"], train["3_preds"])))
 print("4th model score: ",np.sqrt(mean_squared_error(train["target"], train["4_preds"])))
 print("Blending model score: ",np.sqrt(mean_squared_error(train["target"], train["4_preds"]*0.1+train["3_preds"]*0.3+train["2_preds"]*0.3+train["1_preds"]*0.3)) )<create_dataframe>
emb_szs = data.get_emb_szs({})
 model = TabularModel(emb_szs, len(data.cont_names), out_sz=data.c, layers=[200,100], ps=None, emb_drop=0., y_range=None, use_bn=True)
 learn = Learner(data, model, loss_func=KappaLoss() , path='/tmp' )
input_df = train[["1_preds","2_preds","3_preds","4_preds"]].copy()
sub_input_df = test[["1_preds","2_preds","3_preds","4_preds"]].copy()<train_model>
learn.fit(10, 5e-2)
 pred = learn.get_preds(ds_type=DatasetType.Train)
 y_pred = [int(np.argmax(row)) for row in pred[0]]
print('QWK insample', cohen_kappa_score(y_pred, pred[1], weights='quadratic'))
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_stack = np.zeros(len(train))
score_list= []
fold = 1
test_preds_stack = []
for train_index, test_index in kf.split(input_df):
X_train, X_val = input_df.iloc[train_index], input_df.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
reg = LinearRegression(fit_intercept=True ).fit(X_train, y_train)
 y_stack = reg.predict(X_val)
 oof_stack[test_index] = y_stack*1
score = np.sqrt(mean_squared_error(y_val, oof_stack[test_index]))
score_list.append(score)
 test_preds_stack.append(reg.predict(sub_input_df.values ).ravel())
 np.sqrt(mean_squared_error(y, oof_stack))<compute_test_metric>
print('RMSE insample', np.sqrt(np.mean(( np.array(y_pred)- np.array(pred[1])) **2)) )
print(np.mean(score_list))
score_list<define_variables>
np.unique(y_pred, return_counts=True )
<train_model><EOS> 
pred = learn.get_preds(ds_type=DatasetType.Test)
 y_pred = [int(np.argmax(row)) for row in pred[0]]
test_csv['AdoptionSpeed'] = y_pred
test_csv[['PetID', 'AdoptionSpeed']].to_csv('submission.csv', index=False )