Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<split>
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore' )
%%time
train = all_df[:train.shape[0]]
test = all_df[train.shape[0]:]<count_missing_values>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
train.isna().sum().sum() , test.isna().sum().sum()<data_type_conversions>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
%%time
train_data = train.drop('target', axis=1 ).to_numpy()
train_target = train['target'].to_numpy()
test_data = test.drop('target', axis=1 ).to_numpy()<drop_column>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
%%time
categorical = all_df.drop(['target'] + continuous,
axis=1 ).columns
cat_cols_idx, cont_cols_idx = list() , list()
for idx, column in enumerate(all_df.drop('target',
axis=1 ).columns):
if column in categorical:
cat_cols_idx.append(idx)
 elif column in continuous:
cont_cols_idx.append(idx )<compute_test_metric>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)] for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):
if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a, rater_b = y, y_pred
min_rating, max_rating = None, None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator, denominator = 0.0, 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train_data, train_target, test_data<import_modules>
user_ID = train.groupby('RescuerID')['PetID'].count().sort_values().reset_index()
user_ID['class'] =(user_ID['PetID']//3 ).clip(0,5)
 user_ID.loc[user_ID['class'] == 3, 'class'] = 2
user_ID.loc[user_ID['class'] == 4, 'class'] = 2
import torch
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler<set_options>
def get_adoptionspeed(arr):
n = arr.shape[0]
c0,c1,c2,c3 = np.int(n*0.05), np.int(n*0.25),np.int(n*0.5), np.int(n*0.75)
 arr = np.argsort(np.argsort(arr))
arr = [0 if x < c0
else 1 if x < c1
else 2 if x < c2
else 3 if x < c3
else 4 for x in arr]
return arr
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministick = True
torch.backends.cudnn.benchmark = False
set_seed(27 )<data_type_conversions>
def kappa(preds, train_data):
labels = train_data.get_label()
pred_class = get_adoptionspeed(preds)
 return 'kappa', quadratic_weighted_kappa(labels, pred_class), True
class ClassificationDataset(Dataset):
def __init__(self, data, targets=None,
is_train=True, cat_cols_idx=None,
cont_cols_idx=None):
self.data = data
self.targets = targets
self.is_train = is_train
self.cat_cols_idx = cat_cols_idx
self.cont_cols_idx = cont_cols_idx
def __getitem__(self, idx):
row = self.data[idx].astype('float32')
 data_cat = []
data_cont = []
result = None
if self.cat_cols_idx:
data_cat = torch.tensor(row[self.cat_cols_idx])
 if self.cont_cols_idx:
data_cont = torch.tensor(row[self.cont_cols_idx])
 data = [data_cat, data_cont]
if self.is_train:
result = {'data': data,
'target': torch.tensor(self.targets[idx])}
else:
result = {'data': data}
return result
def __len__(self):
return(len(self.data))<create_dataframe>
all_data = pd.concat([train[test.columns], test], ignore_index=True )
train_dataset = ClassificationDataset(train_data,
targets=train_target,
cat_cols_idx=cat_cols_idx,
cont_cols_idx=cont_cols_idx)
 test_dataset = ClassificationDataset(test_data,
cat_cols_idx=cat_cols_idx,
cont_cols_idx=cont_cols_idx,
is_train=False )<split>
train['Fee_v1'] =(train['Fee'] > 0 ).astype('int')
 train['Sterilized'].replace(3, np.nan, inplace = True)
 train['Vaccinated'].replace(3, np.nan, inplace = True)
 test['Fee_v1'] =(test['Fee'] > 0 ).astype('int')
 test['Sterilized'].replace(3, np.nan, inplace = True)
 test['Vaccinated'].replace(3, np.nan, inplace = True )
def split_dataset(trainset, valid_size=0.2, batch_size=64):
num_train = len(trainset)
 indices = list(range(num_train))
np.random.shuffle(indices)
 split = int(np.floor(valid_size * num_train))
valid_idx, train_idx = indices[:split], indices[split:]
valid_sampler = SubsetRandomSampler(valid_idx)
 train_sampler = SubsetRandomSampler(train_idx)
 valid_loader = DataLoader(trainset,
batch_size=batch_size,
sampler=valid_sampler)
 train_loader = DataLoader(trainset,
batch_size=batch_size,
sampler=train_sampler)
 return train_loader, valid_loader<split>
user_ID.columns = ['RescuerID', 'cntPet','class']
train = train.merge(user_ID, on=['RescuerID'], how='left')
 test = test.merge(user_ID, on=['RescuerID'], how='left' )
train_loader, valid_loader = split_dataset(train_dataset,
valid_size=0.2,
batch_size=2000 )<choose_model_class>
labels_breed['First'] = labels_breed['BreedName'].str[0]
labels_breed.columns = ['Breed1','Type','BreedName','First']
all_data = all_data.merge(labels_breed[['Breed1','First']], on=['Breed1'], how='left' )
class ClassificationEmbdNN(torch.nn.Module):
def __init__(self, emb_dims, no_of_cont=None):
super(ClassificationEmbdNN, self ).__init__()
self.emb_layers = torch.nn.ModuleList([torch.nn.Embedding(x, y)
 for x, y in emb_dims])
 no_of_embs = sum([y for x, y in emb_dims])
 self.no_of_embs = no_of_embs
self.emb_dropout = torch.nn.Dropout(0.2)
 self.no_of_cont = 0
if no_of_cont:
self.no_of_cont = no_of_cont
self.bn_cont = torch.nn.BatchNorm1d(no_of_cont)
 self.fc1 = torch.nn.Linear(in_features=self.no_of_embs + self.no_of_cont,
out_features=256)
 self.dropout1 = torch.nn.Dropout(0.2)
 self.bn1 = torch.nn.BatchNorm1d(256)
 self.act1 = torch.nn.ReLU()
self.fc2 = torch.nn.Linear(in_features=256,
out_features=256)
 self.dropout2 = torch.nn.Dropout(0.2)
 self.bn2 = torch.nn.BatchNorm1d(256)
 self.act2 = torch.nn.ReLU()
self.fc3 = torch.nn.Linear(in_features=256,
out_features=64)
 self.dropout3 = torch.nn.Dropout(0.2)
 self.bn3 = torch.nn.BatchNorm1d(64)
 self.act3 = torch.nn.ReLU()
self.fc4 = torch.nn.Linear(in_features=64,
out_features=1)
 self.act4 = torch.nn.Sigmoid()
def forward(self, x_cat, x_cont=None):
if self.no_of_embs != 0:
x = [emb_layer(x_cat[:, i])
 for i, emb_layer in enumerate(self.emb_layers)]
x = torch.cat(x, 1)
 x = self.emb_dropout(x)
 if self.no_of_cont != 0:
x_cont = self.bn_cont(x_cont)
 if self.no_of_embs != 0:
x = torch.cat([x, x_cont], 1)
 else:
x = x_cont
x = self.fc1(x)
 x = self.dropout1(x)
 x = self.bn1(x)
 x = self.act1(x)
 x = self.fc2(x)
 x = self.dropout2(x)
 x = self.bn2(x)
 x = self.act2(x)
 x = self.fc3(x)
 x = self.dropout3(x)
 x = self.bn3(x)
 x = self.act3(x)
 x = self.fc4(x)
 x = self.act4(x)
 return x<train_model>
more_dim = 0
lb = LabelBinarizer(sparse_output=True)
 X_Breed1 = lb.fit_transform(all_data['Breed1'])
 more_dim += X_Breed1.shape[1]
X_Breed2 = lb.fit_transform(all_data['Breed1'])
 more_dim += X_Breed2.shape[1]
X_State = lb.fit_transform(all_data['State'])
 more_dim += X_State.shape[1]
X_Color1 = lb.fit_transform(all_data['Color1'])
 more_dim += X_Color1.shape[1]
X_Color2 = lb.fit_transform(all_data['Color2'])
 more_dim += X_Color2.shape[1]
X_Color3 = lb.fit_transform(all_data['Color3'])
 more_dim += X_Color3.shape[1]
def train_network(model, train_loader, valid_loader,
loss_func, optimizer, n_epochs=20,
saved_model='model.pt'):
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 model.to(device)
 train_losses = list()
valid_losses = list()
valid_loss_min = np.Inf
for epoch in range(n_epochs):
train_loss = 0.0
valid_loss = 0.0
train_auc = 0.0
valid_auc = 0.0
model.train()
for batch in tqdm(train_loader):
optimizer.zero_grad()
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float))
loss = loss_func(output, batch['target'].to(device,
dtype=torch.float))
loss.backward()
optimizer.step()
train_auc += metrics.roc_auc_score(batch['target'].cpu().numpy() ,
output.detach().cpu().numpy())
 train_loss += loss.item() * batch['data'][0].size(0)
 model.eval()
for batch in tqdm(valid_loader):
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float))
loss = loss_func(output, batch['target'].to(device,
dtype=torch.float))
valid_auc += metrics.roc_auc_score(batch['target'].cpu().numpy() ,
output.detach().cpu().numpy())
 valid_loss += loss.item() * batch['data'][0].size(0)
 train_loss = np.sqrt(train_loss / len(train_loader.sampler.indices))
valid_loss = np.sqrt(valid_loss / len(valid_loader.sampler.indices))
train_auc = train_auc / len(train_loader)
 valid_auc = valid_auc / len(valid_loader)
 train_losses.append(train_loss)
 valid_losses.append(valid_loss)
 print('Epoch: {}.Training loss: {:.6f}.Validation loss: {:.6f}' 
.format(epoch, train_loss, valid_loss))
print('Training AUC: {:.6f}.Validation AUC: {:.6f}' 
.format(train_auc, valid_auc))
if valid_loss < valid_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...' 
.format(valid_loss_min, valid_loss))
torch.save(model.state_dict() , saved_model)
 valid_loss_min = valid_loss
return train_losses, valid_losses 
<count_unique_values>
res = []
for i in tqdm_notebook(train_sentiment_files + test_sentiment_files):
with open(i, 'r', encoding='utf-8')as f:
tmp = json.load(f)
 res.append(( i[-14:-5], len(tmp['sentences']), len(tmp['tokens']), \
tmp['documentSentiment']['magnitude'], tmp['documentSentiment']['score']))
tmp = pd.DataFrame(res)
 tmp.columns = ['PetID', 'len_des','n_tokens','magnitude','score']
train = train.merge(tmp, on=['PetID'], how='left')
 test = test.merge(tmp, on=['PetID'], how='left' )
cat_dim = [int(all_df[col].nunique())for col in categorical]
cat_dim = [[x, min(200,(x + 1)// 2)] for x in cat_dim]
for el in cat_dim:
if el[0] < 10:
el[1] = el[0]
cat_dim<choose_model_class>
tmp = all_data.groupby(['RescuerID'])['Age','Fee','PhotoAmt','VideoAmt','Quantity']\ 
.agg(['min','max','mean','sum'] ).reset_index()
tmp.columns = ['RescuerID','min_age','max_age','mean_age','sum_age',\
'min_Fee','max_Fee','mean_Fee','sum_Fee',\
'min_PhotoAmt','max_PhotoAmt','mean_PhotoAmt','sum_PhotoAmt',\
'min_VideoAmt','max_VideoAmt','mean_VideoAmt','sum_VideoAmt',\
'min_Quantity','max_Quantity','mean_Quantity','sum_Quantity']
train = train.merge(tmp, on=['RescuerID'], how='left')
 test = test.merge(tmp, on=['RescuerID'], how='left')
model = ClassificationEmbdNN(emb_dims=cat_dim,
no_of_cont=len(continuous))
loss_func = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=0.1)
 train_losses, valid_losses = train_network(model=model,
train_loader=train_loader,
valid_loader=valid_loader,
loss_func=loss_func,
optimizer=optimizer,
n_epochs=3,
saved_model='simple_nn.pt' )<predict_on_test>
tmp = all_data[all_data['MaturitySize'] > 0].groupby(['RescuerID'])['MaturitySize'].agg(['min','max','mean'] ).reset_index()
tmp.columns = ['RescuerID','min_MaturitySize_1','max_MaturitySize_1','mean_MaturitySize_1']
train = train.merge(tmp, on=['RescuerID'], how='left')
 test = test.merge(tmp, on=['RescuerID'], how='left' )
def predict(data_loader, model):
model.eval()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 model.to(device)
 with torch.no_grad() :
predictions = None
for i, batch in enumerate(tqdm(data_loader)) :
output = model(batch['data'][0].to(device,
dtype=torch.long),
batch['data'][1].to(device,
dtype=torch.float)).cpu().numpy()
if i == 0:
predictions = output
else:
predictions = np.vstack(( predictions, output))
return predictions<predict_on_test>
tmp = all_data[all_data['Health'] > 0].groupby(['RescuerID'])['MaturitySize'].agg(['min','max','mean'] ).reset_index()
tmp.columns = ['RescuerID','min_Health','max_Health','mean_Health']
train = train.merge(tmp, on=['RescuerID'], how='left')
 test = test.merge(tmp, on=['RescuerID'], how='left' )
model.load_state_dict(torch.load('simple_nn.pt'))
test_loader = DataLoader(test_dataset,
batch_size=1000)
 nn_predictions = predict(test_loader, model )<create_dataframe>
count_cols = []
for i in ['Breed1','Color1','Sterilized','Vaccinated','MaturitySize','FurLength','Health']:
tmp = all_data.groupby(['RescuerID'])[i].value_counts().unstack().replace(np.nan, 0 ).reset_index()
tmp.columns = [tmp.columns[0]] + ['C_{}_C_{}'.format(i, str(j)) for j in range(len(tmp.columns)- 1)]
count_cols = count_cols + list(tmp.columns[1:])
 train = train.merge(tmp, on = ['RescuerID'], how='left')
 test = test.merge(tmp, on = ['RescuerID'], how='left' )
nn_predictions_df = pd.DataFrame({'id': sample_submission['id'], 'target': nn_predictions.squeeze() } )<import_modules>
tmp = pd.DataFrame({0:pet_id, 1:annot_des} ).groupby([0])[1].apply(lambda x: " ".join(x)).reset_index()
tmp.columns = ['PetID','labelAnnotations_des']
train = train.merge(tmp, on=['PetID'], how='left')
 test = test.merge(tmp, on=['PetID'], how='left')
 stopWords = stopwords.words('english')
 tfidf = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
min_df = 4,
token_pattern=r'\w{1,}',
stop_words= stopWords,
ngram_range=(1, 1),
max_features=30000)
 vectorizer = tfidf.fit(train['labelAnnotations_des'].fillna("NULL"))
train_labelAnnotations_des = tfidf.transform(train['labelAnnotations_des'].fillna("NULL"))
test_labelAnnotations_des = tfidf.transform(test['labelAnnotations_des'].fillna("NULL"))
from catboost import CatBoostClassifier<split>
tmp = train.groupby(['RescuerID'])['len_des'].agg(['min','max','mean'] ).reset_index()
tmp.columns = ['RescuerID','min_len_des','max_len_des','mean_len_des']
train = train.merge(tmp, on=['RescuerID'], how='left')
 tmp = test.groupby(['RescuerID'])['len_des'].agg(['min','max','mean'] ).reset_index()
tmp.columns = ['RescuerID','min_len_des','max_len_des','mean_len_des']
test = test.merge(tmp, on=['RescuerID'], how='left' )
X = train.drop('target', axis=1)
 y = train['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=27)
 test_data = test.drop('target', axis=1 )<define_variables>
countvectorizer = CountVectorizer(min_df=4)
 vectorizer = countvectorizer.fit(all_data['Description'].fillna("NULL"))
train_count_des = countvectorizer.transform(train['Description'].fillna("NULL"))
test_count_des = countvectorizer.transform(test['Description'].fillna("NULL"))
cat_features = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_5', 'nom_6', 'nom_7',
'nom_8', 'nom_9', 'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4',
'ord_5_1', 'ord_5_2', 'nan_features', 'day_0', 'day_1', 'day_2',
'day_3', 'day_4', 'day_5', 'day_6', 'day_7', 'month_0', 'month_1',
'month_2', 'month_3', 'month_4', 'month_5', 'month_6', 'month_7',
'month_8', 'month_9', 'month_10', 'month_11', 'month_12', 'nom_0_0',
'nom_0_1', 'nom_0_2', 'nom_0_3', 'nom_1_0', 'nom_1_1', 'nom_1_2',
'nom_1_3', 'nom_1_4', 'nom_1_5', 'nom_1_6', 'nom_2_0', 'nom_2_1',
'nom_2_2', 'nom_2_3','nom_2_4', 'nom_2_5', 'nom_2_6', 'nom_3_0',
'nom_3_1', 'nom_3_2','nom_3_3', 'nom_3_4', 'nom_3_5', 'nom_3_6',
'nom_4_0', 'nom_4_1', 'nom_4_2', 'nom_4_3', 'nom_4_4']<choose_model_class>
countvectorizer = CountVectorizer(min_df=4)
 vectorizer = countvectorizer.fit(all_data['Name'].fillna("NULL"))
train_count_name = countvectorizer.transform(train['Name'].fillna("NULL"))
test_count_name = countvectorizer.transform(test['Name'].fillna("NULL"))
best_params = {
'bagging_temperature': 0.8,
'depth': 5,
'iterations': 1000,
'l2_leaf_reg': 30,
'learning_rate': 0.05,
'random_strength': 0.8
}
model_cat = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric='AUC',
nan_mode='Min',
thread_count=4,
task_type='GPU',
verbose=True)
 model_cat.fit(X_train, y_train,
eval_set=(X_test, y_test),
cat_features=cat_features,
verbose_eval=300,
early_stopping_rounds=500,
use_best_model=True,
plot=False)
<predict_on_test>
BatchNormalization, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, Conv1D, Multiply, Add,Flatten, Ave
def top_2_accuracy(x,y): return top_k_categorical_accuracy(x,y, 2 )
cat_predictions = model_cat.predict_proba(test_data)[:, 1]<create_dataframe>
img_feats = np.load(".. /input/darling-lgbm-output-img-feats/img_feats.npy" )
cat_predictions_df = pd.DataFrame({'id': sample_submission['id'],
'target': cat_predictions} )<prepare_output>
img_id = pd.read_csv(".. /input/darling-lgbm-output-img-feats/metadata_img.csv" )
res_sub = pd.DataFrame({'id': sample_submission['id']})
 res_sub.head()<feature_engineering>
lb = LabelEncoder()
img_id['Breed1'] = lb.fit_transform(img_id['Breed1'] )
res_sub['target'] = round(( cat_predictions_df['target'] + nn_predictions_df['target'])/ 2, 2 )<save_to_csv>
def build_model(state, img_feats_xception):
x1 = Embedding(img_id['Breed1'].max() + 1, 10,trainable= True )(state)
 x = Dense(196, activation="sigmoid",kernel_initializer=he_uniform(seed=0))(img_feats_xception)
 x = concatenate([Flatten()(x1),x])
 x = Dense(5, activation='softmax',kernel_initializer=he_uniform(seed=0))(x)
 return x
res_sub.to_csv('res_sub.csv', index=False )<set_options>
img_oof_cache = []; f = 0
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=500)
 for dev_index, val_index in(kf.split(user_ID, user_ID['class'])) :
dev_user, val_user = user_ID.iloc[dev_index].RescuerID, user_ID.iloc[val_index].RescuerID
X_train_idx, X_val_idx = list(img_id[img_id['RescuerID'].isin(dev_user)].index.values),\
list(img_id[img_id['RescuerID'].isin(val_user)].index.values)
 img_feats_xception = Input(( 2048,), name="i1")
 state = Input(shape=[1], name="i2")
 output_layer = build_model(state, img_feats_xception)
 model = Model([state, img_feats_xception], output_layer)
 model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['categorical_accuracy', top_2_accuracy])
 if(f == 0):
model.summary()
f += 1
dev_img_feats, dev_other_feats = img_feats[X_train_idx], img_id.iloc[X_train_idx]
val_img_feats, val_other_feats = img_feats[X_val_idx], img_id.iloc[X_val_idx]
model.fit({"i1":dev_img_feats,"i2":dev_other_feats['Breed1']}, to_categorical(dev_other_feats['AdoptionSpeed'].values),\
batch_size=32, verbose=2,shuffle=True,epochs=30,\
validation_data=({"i1":val_img_feats,"i2":val_other_feats['Breed1']}, to_categorical(val_other_feats['AdoptionSpeed'].values)))
 img_oof = pd.DataFrame()
img_oof['PetID'] = val_other_feats['PetID'].values
val_pres = model.predict({"i1":val_img_feats,"i2":val_other_feats['Breed1']}, batch_size=128, verbose=2)
 for i in range(5):
img_oof['target_img_' + str(i)] = val_pres[:,i]
img_oof_cache.append(img_oof )
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 50
BIN_COL = [f'bin_{i}' for i in range(5)]
NOM_COL = [f'nom_{i}' for i in range(10)]
ORD_COL = [f'ord_{i}' for i in range(6)]
NOM_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
NOM_0_4 = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']
DATE_COL = ['day','month'] 
<load_from_csv>
img_oof = pd.concat(img_oof_cache, ignore_index=True)
 img_oof = img_oof.groupby(['PetID'] ).mean().reset_index()
for i in range(5):
c = 'target_img_' + str(i)
 if c in train.columns:
train.drop([c], axis=1, inplace = True)
 train = train.merge(img_oof[['PetID',c]], on=['PetID'], how='left')
 test[c] = 0
%%time
def read_csv() :
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 train_id = train['id']
test_id = test['id']
train.drop('id', axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 return train, test, train_id, test_id
def preprocessing(df):
df.bin_3.replace({'F':0, 'T':1}, inplace=True)
 df.bin_4.replace({'N':0, 'Y':1}, inplace=True)
 ord_1_map = {'Novice':1,'Contributor':2,'Expert':3,'Master':4,'Grandmaster':5}
ord_2_map = {'Freezing':1, 'Cold':2,'Warm':3,'Hot':4, 'Boiling Hot':5,'Lava Hot':6}
df.loc[df['ord_1'].notnull() ,'ord_1'] = df.loc[df['ord_1'].notnull() ,'ord_1'].map(ord_1_map)
 df.loc[df['ord_2'].notnull() ,'ord_2'] = df.loc[df['ord_2'].notnull() ,'ord_2'].map(ord_2_map)
 df.loc[df['ord_3'].notnull() ,'ord_3'] = df.loc[df['ord_3'].notnull() ,'ord_3'].apply(
 lambda c: ord(c)- ord('a')+ 1)
 df.loc[df['ord_4'].notnull() ,'ord_4'] = df.loc[df['ord_4'].notnull() ,'ord_4'].apply(
 lambda c: ord(c)- ord('A')+ 1)
 for col in ['ord_1','ord_2','ord_3','ord_4',]:
df[col] = df[col].astype(np.float32)
 df.day = df.day.replace({3:5,2:6,1:7})
 df.loc[df.ord_5.notnull() , 'ord_5_1'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[0])
 df.loc[df.ord_5.notnull() , 'ord_5_2'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[1])
 df.loc[df['ord_5_1'].notnull() ,'ord_5_1'] = df.loc[df['ord_5_1'].notnull() ,'ord_5_1'].apply(
 lambda c: ord(c)- ord('a')+ 33 ).astype(np.float32)
 df.loc[df['ord_5_2'].notnull() ,'ord_5_2'] = df.loc[df['ord_5_2'].notnull() ,'ord_5_2'].apply(
 lambda c: ord(c)- ord('a')+ 33)
 return df
def filling_NaN(df):
df.fillna(-1, inplace=True)
 df.day = df.day.astype(int)
 df.month = df.month.astype(int)
 return df
def target_encoding(cols, smoothing=1.0, min_samples_leaf=1):
for col in cols:
encoder = ce.TargetEncoder(cols=col,
smoothing=smoothing,
min_samples_leaf=min_samples_leaf)
 train[f'{col}_mean'] = encoder.fit_transform(train[col], train['target'])[col].astype(np.float32)
 test[f'{col}_mean'] = encoder.transform(test[col])[col].astype(np.float32)
 del encoder
gc.collect()<data_type_conversions>
pca = PCA(n_components=5, random_state = 10)
 tmp = pd.DataFrame(pca.fit_transform(img_feats))
tmp.columns = ['pca_' + str(i)for i in range(5)]
tmp['PetID'] = img_id['PetID']
tmp = tmp.groupby(['PetID'] ).mean().reset_index()
for i in range(5):
c = 'pca_' + str(i)
 if c in train.columns:
train.drop([c], axis=1, inplace = True)
 train = train.merge(tmp[['PetID',c]], on=['PetID'], how='left')
 test[c] = 0
%%time
USE_DUMMIES = BIN_COL+NOM_COL+DATE_COL+['ord_0','ord_1', 'ord_2','ord_3', 'ord_4']
NOT_DUMMIES = [s for s in train.columns if '_mean' in s]
for col in NOT_DUMMIES:
scaler = MaxAbsScaler()
train[col] = scaler.fit_transform(train[[col]] ).flatten().astype(np.float32)
 test[col] = scaler.transform(test[[col]] ).flatten().astype(np.float32)
 del scaler;gc.collect()<concatenate>
train['Description'].str.count(" " )
%%time
print('drop features:',test.columns.drop(USE_DUMMIES+NOT_DUMMIES ).tolist())
 target = train['target']
train = train[USE_DUMMIES+NOT_DUMMIES]
test = test[USE_DUMMIES+NOT_DUMMIES]
traintest = pd.concat([train, test], sort=False)
 traintest = traintest.reset_index(drop=True)
 traintest.head(10 )<categorify>
res = []; n_train = train.shape[0]; output = 0
params = {
'boosting': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.01,
'num_leaves': 30,
'max_depth': 15,
'colsample_bytree': 0.2,
'min_data_in_leaf':50,
'bagging_freq': 10,
'bagging_fraction': 0.7,
'min_data_per_group': 20,
'seed': 100
}
train['len_des_1'] = train['Description'].fillna("" ).astype(str ).str.len()
test['len_des_1'] = test['Description'].fillna("" ).astype(str ).str.len()
train['len_word'] = train['Description'].fillna("" ).str.count(" ")
 test['len_word'] = test['Description'].fillna("" ).str.count(" ")
 train['NoName'] =(( train['Name'].fillna("NULL" ).str.contains("No Name")) | train['Name'].isnull() ).astype('int')
 test['NoName'] =(( test['Name'].fillna("NULL" ).str.contains("No Name")) | test['Name'].isnull() ).astype('int')
 kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=500)
 predictors = ['magnitude','Age','Type','Breed1','Breed2','Color1','Color2','Color3','PhotoAmt','VideoAmt','State',\
'Sterilized','Vaccinated','Quantity','Fee','Fee_v1','MaturitySize', 'Gender','FurLength','Health',\
'cntPet','len_des','len_des_1','min_age','max_age','mean_age','min_Fee','max_Fee',\
'mean_Fee','min_PhotoAmt','max_PhotoAmt','mean_PhotoAmt',\
'min_Quantity','max_Quantity','mean_Quantity','min_VideoAmt','max_VideoAmt','mean_VideoAmt',
'min_MaturitySize_1','max_MaturitySize_1','mean_MaturitySize_1','min_Health','max_Health','mean_Health',\
'min_score_labelAnnotations','max_score_labelAnnotations','mean_score_labelAnnotations',\
'min_score_labelAnnotations_1','max_score_labelAnnotations_1','mean_score_labelAnnotations_1',\
'min_x','max_x', 'mean_x','min_y','max_y','mean_y',
'min_pixelscore','max_pixelscore', 'mean_pixelscore','min_pixelFraction','max_pixelFraction','mean_pixelFraction',\
'min_red_0','max_red_0', 'mean_red_0','min_green_0','max_green_0','mean_green_0',\
'min_blue_0','max_blue_0', 'mean_blue_0',
'min_len_des','max_len_des','mean_len_des',
] + list(count_cols)
 categorical = ['Type','State','Gender','Breed1','Breed2','Color1','Color2','Color3','Fee_v1','MaturitySize', 'FurLength','Health']
Xspr_train = csr_matrix(hstack([csr_matrix(train[predictors]), train_count_des, train_labelAnnotations_des,\
X_Breed1[:n_train], X_Breed2[:n_train], X_State[:n_train],\
X_Color1[:n_train], X_Color2[:n_train], X_Color3[:n_train]]))
Xspr_test = csr_matrix(hstack([csr_matrix(test[predictors]), test_count_des, test_labelAnnotations_des,\
X_Breed1[n_train:], X_Breed2[n_train:], X_State[n_train:],\
X_Color1[n_train:], X_Color2[n_train:], X_Color3[n_train:]]))
for dev_index, val_index in(kf.split(user_ID, user_ID['class'])) :
dev_user, val_user = user_ID.iloc[dev_index].RescuerID, user_ID.iloc[val_index].RescuerID
X_train_idx, X_val_idx = list(train[train['RescuerID'].isin(dev_user)].index.values),\
list(train[train['RescuerID'].isin(val_user)].index.values)
 X_val = train.iloc[X_val_idx]
print(len(X_train_idx), len(X_val_idx))
dtrain = lgb.Dataset(Xspr_train[X_train_idx], label=train.iloc[X_train_idx]['AdoptionSpeed'].values,
feature_name=predictors+ ['tfidf_' + str(i)for i in range(\
train_count_des.shape[1]+ train_labelAnnotations_des.shape[1])] +[str(i)for i in range(more_dim)],
categorical_feature=categorical 
)
 dvalid = lgb.Dataset(Xspr_train[X_val_idx], label=train.iloc[X_val_idx]['AdoptionSpeed'].values,
feature_name=predictors + ['tfidf_' for i in range(
 train_count_des.shape[1] + train_labelAnnotations_des.shape[1])] +[str(i)for i in range(more_dim)],
categorical_feature=categorical 
)
 lgb_model = lgb.train(params, dtrain,
valid_sets=[dtrain, dvalid], valid_names=['train','valid'],
num_boost_round= 3000,early_stopping_rounds=300,
verbose_eval=100,)
 oof = pd.DataFrame()
oof['PetID'] = X_val['PetID']
oof['target'] = X_val['AdoptionSpeed']
oof['preds'] = lgb_model.predict(Xspr_train[X_val_idx], num_iteration=lgb_model.best_iteration)
 res.append(oof)
 output += lgb_model.predict(Xspr_test, num_iteration=lgb_model.best_iteration)
 oof = pd.concat(res, ignore_index=True)
 quadratic_weighted_kappa(oof['target'], get_adoptionspeed(oof['preds']))
%%time
traintest = pd.get_dummies(traintest,
columns=USE_DUMMIES,
dummy_na=False,
drop_first=False,
sparse=True,
dtype=np.int8)
 traintest.head(10 )<data_type_conversions>
submission = pd.DataFrame({'PetID': test['PetID'].values, 'RescuerID': test['RescuerID'].values, \
'AdoptionSpeed': get_adoptionspeed(output)} )
%%time
def convert_sparse(traintest, train_length):
train_ohe = traintest.iloc[:train_length, :]
test_ohe = traintest.iloc[train_length:, :]
train_ohe1 = scipy.sparse.bsr_matrix(train_ohe[NOT_DUMMIES])
 test_ohe1 = scipy.sparse.bsr_matrix(test_ohe[NOT_DUMMIES])
 train_ohe2 = scipy.sparse.csr_matrix(train_ohe.drop(columns=NOT_DUMMIES))
test_ohe2 = scipy.sparse.csr_matrix(test_ohe.drop(columns=NOT_DUMMIES))
train_ohe = scipy.sparse.hstack([train_ohe1,train_ohe2] ).tocsr()
test_ohe = scipy.sparse.hstack([test_ohe1, test_ohe2] ).tocsr()
return train_ohe, test_ohe
train_ohe, test_ohe = convert_sparse(traintest, train_length=len(train))
train_ohe.shape, test_ohe.shape<set_options>
tmp = submission.groupby(['RescuerID'])['AdoptionSpeed'].apply(lambda x: x.mode() [0] ).reset_index()
submission.drop(['AdoptionSpeed'], axis=1, inplace = True)
 submission = submission.merge(tmp, on=['RescuerID'], how='left' )
del train, test;gc.collect()<categorify>
submission[['PetID','AdoptionSpeed']].to_csv('submission.csv', index=False )
<compute_train_metric><EOS> 
submission['AdoptionSpeed'].value_counts()