Kaggle competition: Titanic - Machine Learning from Disaster
text_features.append(doc_df )<merge>
import re
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold
from collections import Counter
from imblearn.under_sampling import RandomUnderSampler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import *
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from pprint import pprint
import xgboost
from sklearn.model_selection import cross_val_score, GridSearchCV, KFold, RandomizedSearchCV
from sklearn.metrics import auc, accuracy_score, confusion_matrix, mean_squared_error
from xgboost import XGBClassifier
import time
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
X_text_char = data_df[['PetID','Name','BreedName_full']].merge(metadata_desc[['PetID','meta_annots_top_desc']], on='PetID', how='left')
 X_text_char['meta_annots_top_desc'].fillna(' ',inplace=True)
 for c in ['Name','BreedName_full','meta_annots_top_desc']:
svd_ = TruncatedSVD(
 n_components=16, random_state=1337)
 tfidf_col = TfidfVectorizer(ngram_range =(1,5), analyzer='char',
strip_accents='unicode' ).fit_transform(X_text_char[c])
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_CHAR_{}_'.format(c))
text_features.append(svd_col)
 del X_text_char
gc.collect()<concatenate>
def get_cv_roc_pr_rc(clf, skf, X, y):
roc_auc = []
pr_auc = []
precision = []
recall = []
models = []
for train, test in skf.split(X, y):
models.append(clf.fit(X.iloc[train,:], y.iloc[train]))
y_pred = clf.predict(X.iloc[test,:])
 y_pred_prob = clf.predict_proba(X.iloc[test,:])[:,1]
roc_auc.append(roc_auc_score(y.iloc[test], y_pred_prob))
pr_auc.append(average_precision_score(y.iloc[test], y_pred_prob))
precision.append(precision_score(y.iloc[test], y_pred))
recall.append(recall_score(y.iloc[test], y_pred))
print('ROC-AUC:', np.mean(roc_auc))
print('PR-AUC:', np.mean(pr_auc))
print('Precission:', np.mean(precision))
print('Recall:', np.mean(recall))
return roc_auc, pr_auc, precision, recall, models
text_features = pd.concat(text_features, axis=1 )<feature_engineering>
def get_cv_roc_pr_rc_2(clf, skf, X, y):
roc_auc = []
pr_auc = []
precision = []
recall = []
models = []
for train, test in skf.split(X, y):
models.append(clf.fit(X.iloc[train,:], y.iloc[train]))
y_pred = clf.predict(X.iloc[test,:])
 roc_auc.append(roc_auc_score(y.iloc[test], y_pred))
pr_auc.append(average_precision_score(y.iloc[test], y_pred))
precision.append(precision_score(y.iloc[test], y_pred))
recall.append(recall_score(y.iloc[test], y_pred))
print('ROC-AUC:', np.mean(roc_auc))
print('PR-AUC:', np.mean(pr_auc))
print('Precission:', np.mean(precision))
print('Recall:', np.mean(recall))
return roc_auc, pr_auc, precision, recall, models
text_features['PetID'] = X_text['PetID'].values<feature_engineering>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
def get_lda_feature(data, target, source, name, n_topic = 10, random_state = 999):
retval = pd.DataFrame(data[[target, source]])
 x = retval.groupby(target, as_index=False)[source].agg({'list':(lambda x: list(x)) })
 x['sentence'] = x['list'].apply(lambda x: list(map(str,x)))
 docs = x['sentence'].tolist()
dictionary = corpora.Dictionary(docs)
 corpus = [dictionary.doc2bow(text)for text in docs]
lda = LdaMulticore(corpus, id2word=dictionary, num_topics=n_topic, random_state = random_state)
 docres = [dict(lda[doc_bow])for doc_bow in corpus]
doc_df = pd.DataFrame(docres,dtype=np.float16 ).fillna(0.001)
 doc_df.columns = ['%s_lda_%d'%(name,x)for x in range(n_topic)]
doc_df[target] = x[target]
return doc_df<feature_engineering>
data = pd.concat([train.drop(['Survived'], axis=1), test] )
breed_color_lda = get_lda_feature(data_df, 'Breed1', 'Color_full', 'breed', n_topic=5)
 breed_breed_lda = get_lda_feature(data_df, 'Breed1', 'Breed2', 'breed_breed', n_topic=5)
 state_breed_lda = get_lda_feature(data_df, 'State', 'Breed_full', 'State_breed', n_topic=5)
 state_color_lda = get_lda_feature(data_df, 'State', 'Color_full', 'State_color',n_topic=5)
<merge>
train.groupby('Pclass' ).Survived.mean()
data_df_proc = data_df.copy()
data_df_proc = data_df_proc.merge(senti_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(meta_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(feat_df, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_resnet50_features[['PetID','resnet_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_dense_features[['PetID','dense_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_resnet34_features[['PetID','res34_%d'%IMG_FEATURE_DIM_NN2]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(dnn_dense121_features[['PetID','dense121_%d'%IMG_FEATURE_DIM_NN]], how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_resnet34_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_resnet50_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_dense_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(svd_dense121_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(cluster_img_features, how='left', on='PetID')
 data_df_proc = data_df_proc.merge(breed_color_lda, how='left', on='Breed1')
 data_df_proc = data_df_proc.merge(breed_breed_lda, how='left', on='Breed1')
 data_df_proc = data_df_proc.merge(state_breed_lda, how='left', on='State')
 data_df_proc = data_df_proc.merge(state_color_lda, how='left', on='State')
 data_df_proc = data_df_proc.merge(text_features, how='left', on='PetID')
 print(data_df_proc.shape)
 assert data_df_proc.shape[0] == data_df.shape[0]<define_variables>
train.groupby(['Pclass', 'Sex'] ).Survived.mean()
early_stop = 300
verbose_eval = 100
num_rounds = 10000
to_drop_columns = ['PetID', 'Name', 'RescuerID', 'AdoptionSpeed', 'target2',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName',
'Description', 'sentiment_entities', 'meta_annots_top_desc','meta_desc',
'Chinese_desc', 'English_desc','BreedName_full','Breed1Name','Breed2Name']<set_options>
train.loc[train.Fare==0]
torch.manual_seed(1991)
 torch.cuda.manual_seed(1991)
 torch.backends.cudnn.deterministic = True<data_type_conversions>
data['Fam_Size'] = data.Parch + data.SibSp + 1
data['Fam_type'] = pd.cut(data.Fam_Size, [0,1,4,7,11], labels=['Solo', 'Small', 'Big', 'Very big'] )
fm_cols = ['Type','age_in_year','Breed1','Breed2','Gender','Color1','Color2','Color3','MaturitySize',
'FurLength','Vaccinated','Dewormed','Sterilized','Health','State','Breed_full',
'Color_full', 'hard_interaction','img_CLUSTER_0']
fm_data = data_df_proc[fm_cols]
fm_values = []
for c in fm_cols:
fm_data.loc[:,c] = fm_data[c].fillna(0)
 fm_data.loc[:,c] = c+'_'+fm_data[c].astype(str)
 fm_values+=fm_data[c].unique().tolist()<categorify>
data['Ticket_lett'] = data.Ticket.apply(lambda x: x[:2])
 data['Ticket_len'] = data.Ticket.apply(lambda x: len(x))
lbe = LabelEncoder()
lbe.fit(fm_values)
 for c in fm_cols:
fm_data.loc[:,c] = lbe.transform(fm_data[c] )<define_variables>
data['Age'] = data['Age'].interpolate(limit_direction='both', method='linear')
 data['Fare'] = data['Fare'].interpolate(limit_direction='both', method='linear')
 data['Embarked'] = data['Embarked'].fillna(data['Embarked'].value_counts().index[0] )
numerical_cols = [x for x in X_train.columns if x not in to_drop_columns+fm_cols+svd_dense_features.columns.tolist() ]<normalization>
data['CatAge'] = pd.qcut(data.Age, q=4, labels=False )
numerical_feats = []
for c in numerical_cols:
numerical_feats.append(data_df_proc[c].fillna(0))
numerical_feats = np.vstack(numerical_feats ).T 
<categorify>
data = data.drop(['PassengerId', 'Name', 'SibSp', 'Parch',
'Ticket', 'Cabin', 'Fam_Size', 'Age'], axis = 1 )
MAX_LEN = 400
class PetDesDataset(Dataset):
def __init__(self, sentences, pos, fm_data, numerical_feat,
mode='train', target=None):
super(PetDesDataset, self ).__init__()
self.data = sentences
self.pos = pos
self.target = target
self.mode = mode
self.fm_data = fm_data
self.fm_dim = fm_data.shape[1]
self.numerical_feat = numerical_feat
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
if index not in range(0, self.__len__()):
return self.__getitem__(np.random.randint(0, self.__len__()))
sentence_len = min(MAX_LEN, len(self.data[index]))
sentence = torch.tensor(self.data[index][:sentence_len])
 fm_data = self.fm_data[index,:]
pos = torch.tensor(self.pos[index][:sentence_len])
 if self.mode != 'test':
return sentence, pos, sentence_len, fm_data, self.numerical_feat[index], self.target[index]
else:
return sentence, pos, sentence_len, fm_data, self.numerical_feat[index]
def nn_collate(batch):
has_label = len(batch[0])== 6
if has_label:
sentences, poses, lengths, fm_data, numerical_feats, label = zip(*batch)
 sentences = nn.utils.rnn.pad_sequence(sentences, batch_first=True ).type(torch.LongTensor)
 poses = nn.utils.rnn.pad_sequence(poses, batch_first=True ).type(torch.LongTensor)
 lengths = torch.LongTensor(lengths)
 fm_data = torch.LongTensor(fm_data)
 numerical_feats = torch.FloatTensor(numerical_feats)
 label = torch.FloatTensor(label)
 return sentences, poses, lengths, fm_data, numerical_feats, label
else:
sentences, poses, lengths, fm_data, numerical_feats = zip(*batch)
 sentences = nn.utils.rnn.pad_sequence(sentences, batch_first=True ).type(torch.LongTensor)
 poses = nn.utils.rnn.pad_sequence(poses, batch_first=True ).type(torch.LongTensor)
 lengths = torch.LongTensor(lengths)
 fm_data = torch.LongTensor(fm_data)
 numerical_feats = torch.FloatTensor(numerical_feats)
 return sentences, poses, lengths, fm_data, numerical_feats<categorify>
data_dun = pd.get_dummies(data, drop_first=True)
 data_dun.shape
def get_mask(sequences_batch, sequences_lengths, cpu=False):
batch_size = sequences_batch.size() [0]
max_length = torch.max(sequences_lengths)
 mask = torch.ones(batch_size, max_length, dtype=torch.float)
 mask[sequences_batch[:, :max_length] == 0] = 0.0
if cpu:
return mask
else:
return mask.cuda()
class Attention(nn.Module):
def __init__(self, feature_dim, bias=True, head_num=1, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.head_num = head_num
weight = torch.zeros(feature_dim, self.head_num)
 bias = torch.zeros(( 1, 1, self.head_num))
nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 self.b = nn.Parameter(bias)
 def forward(self, x, mask=None):
batch_size, step_dim, feature_dim = x.size()
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim, self.head_num)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 if mask is not None:
eij = eij * mask - 99999.9 *(1 - mask)
 a = torch.softmax(eij, dim=1)
 weighted_input = torch.bmm(x.permute(( 0,2,1)) ,
a ).view(batch_size, -1)
 return weighted_input<choose_model_class>
y_train = train['Survived']
embed_size = 300
class FM(nn.Module):
def __init__(self, max_features, feat_len, embed_size):
super(FM, self ).__init__()
self.bias_emb = nn.Embedding(max_features, 1)
 self.fm_emb = nn.Embedding(max_features, embed_size)
 self.feat_len = feat_len
self.embed_size = embed_size
def forward(self, x):
bias = self.bias_emb(x)
 bias = torch.sum(bias,1)
 emb = self.fm_emb(x)
 sum_feature_emb = torch.sum(emb, 1)
 square_sum_feature_emb = sum_feature_emb*sum_feature_emb
square_feature_emb = emb * emb
sum_square_feature_emb = torch.sum(square_feature_emb, 1)
 second_order = 0.5*(square_sum_feature_emb-sum_square_feature_emb)
 return bias+second_order, emb.view(-1, self.feat_len*self.embed_size)
<choose_model_class>
skf = StratifiedKFold(n_splits=5 )
class FmNlpModel(nn.Module):
def turn_on_embedding(self):
self.embedding.weight.requires_grad = True
def __init__(self, hidden_size=64, init_embedding=None, head_num=3,
fm_embed_size=8, fm_feat_len=10, fm_max_feature = 300, numerical_dim = 300,
nb_word = 40000, nb_pos = 200, pos_emb_size = 10):
super(FmNlpModel, self ).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(nb_word, 300, padding_idx=0)
 self.pos_embedding = nn.Embedding(nb_pos+100, pos_emb_size, padding_idx=0)
 if init_embedding is not None:
self.embedding.weight.data.copy_(torch.from_numpy(init_embedding))
self.embedding.weight.requires_grad = False
self.fm = FM(fm_max_feature, fm_feat_len, fm_embed_size)
 self.dropout = nn.Dropout(0.1)
 self.attention_gru = Attention(feature_dim=self.hidden_size * 2, head_num=head_num)
 self.gru = nn.GRU(embed_size+pos_emb_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru2 = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.dnn = nn.Sequential(
 nn.BatchNorm1d(numerical_dim),
nn.Dropout(0.1),
nn.Linear(numerical_dim, 256),
nn.ELU(inplace=True),
nn.Dropout(0.1),
nn.Linear(256, 128),
nn.ELU(inplace=True), 
)
 self.rnn_dnn = nn.Sequential(
 nn.BatchNorm1d(fm_embed_size+fm_feat_len*fm_embed_size +2*head_num * hidden_size+128),
nn.Dropout(0.3),
nn.Linear(fm_embed_size+fm_feat_len*fm_embed_size+2*head_num * hidden_size+128, 32),
nn.ELU(inplace=True), 
)
 self.logit = nn.Sequential(
 nn.Linear(32,1)
)
 def forward(self, x, pos_x, len_x, fm_x, numerical_x):
fm_result, fm_embed = self.fm(fm_x)
 sentence_mask = get_mask(x, len_x)
 x = x * sentence_mask.long()
sentence_mask = torch.unsqueeze(sentence_mask, -1)
 h_embedding = self.embedding(x)
 h_pos_embedding = self.pos_embedding(pos_x)
 h_embedding = torch.cat([h_embedding, h_pos_embedding],2)
 h_embedding = self.dropout(h_embedding)
 sorted_seq_lengths, indices = torch.sort(len_x, descending=True)
 _, desorted_indices = torch.sort(indices, descending=False)
 h_embedding = h_embedding[indices]
packed_inputs = nn.utils.rnn.pack_padded_sequence(h_embedding, sorted_seq_lengths, batch_first=True)
 h_gru, _ = self.gru(packed_inputs)
 h_gru2, _ = self.gru2(h_gru)
 h_gru2, _ = nn.utils.rnn.pad_packed_sequence(h_gru2, batch_first=True)
 h_gru2 = h_gru2[desorted_indices]
att_pool_gru = self.attention_gru(h_gru2, sentence_mask)
 numerical_x = self.dnn(numerical_x)
 x = torch.cat([att_pool_gru,fm_result,fm_embed,numerical_x],1)
 feat = self.rnn_dnn(x)
 out = self.logit(feat)
 return out, feat<prepare_x_and_y>
data_dun_train = data_dun[:891]
data_dun_test = data_dun[891:]
X_train_numerical = numerical_feats[0:len(train)]
X_test_numerical = numerical_feats[len(train):]
X_train_seq = pd.Series(eng_sequences[0:len(train)])
 X_test_seq = pd.Series(eng_sequences[len(train):])
 X_train_pos_seq = pd.Series(pos_sequences[0:len(train)])
 X_test_pos_seq = pd.Series(pos_sequences[len(train):])
 X_train_fm = fm_data.iloc[0:len(train)].values
X_test_fm = fm_data.iloc[len(train):].values
Y_train = data_df.iloc[0:len(train)]['AdoptionSpeed'].values<create_dataframe>
ros = RandomUnderSampler(random_state=0)
 ros.fit(data_dun_train, y_train)
 X_resampled, y_resampled = ros.fit_sample(data_dun_train, y_train )
train_epochs = 6
loss_fn = torch.nn.MSELoss().cuda()
oof_train_nlp = np.zeros(( X_train.shape[0], 32+1))
oof_test_nlp = []
test_set = PetDesDataset(X_test_seq.tolist() , X_test_pos_seq.tolist() , X_test_fm, X_test_numerical, mode='test')
 test_loader = DataLoader(test_set, batch_size=512, shuffle=False, num_workers=1, pin_memory=True,
collate_fn=nn_collate)
 qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
print('fold:', n_fold)
 hist = histogram(Y_train[train_idx].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 training_set = PetDesDataset(X_train_seq[train_idx].tolist() ,
X_train_pos_seq[train_idx].tolist() ,
X_train_fm[train_idx],
X_train_numerical[train_idx], target = Y_train[train_idx])
 validation_set = PetDesDataset(X_train_seq[valid_idx].tolist() ,
X_train_pos_seq[valid_idx].tolist() ,
X_train_fm[valid_idx],
X_train_numerical[valid_idx],target = Y_train[valid_idx])
 training_loader = DataLoader(training_set, batch_size=512, shuffle=True, num_workers=1,
collate_fn=nn_collate)
 validation_loader = DataLoader(validation_set, batch_size=512, shuffle=False, num_workers=1,
collate_fn=nn_collate)
 model = FmNlpModel(hidden_size=48, init_embedding=embedding_matrix, head_num=10,
fm_embed_size=10, fm_feat_len=X_train_fm.shape[1], fm_max_feature=len(fm_values),
numerical_dim=X_train_numerical.shape[1],
nb_word=nb_words, nb_pos=nb_pos, pos_emb_size=10)
 model.cuda()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=4, eta_min=0.0001)
 iteration = 0
min_val_loss = 100
since = time.time()
for epoch in range(train_epochs):
scheduler.step()
model.train()
for sentences, poses, lengths, x_fm, x_numerical, labels in training_loader:
iteration += 1
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
labels = labels.type(torch.FloatTensor ).cuda().view(-1, 1)
 pred,_ = model(sentences, poses, lengths, x_fm, x_numerical)
 loss = loss_fn(pred, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
val_predicts = []
val_feats = []
with torch.no_grad() :
for sentences, poses, lengths, x_fm, x_numerical, labels in validation_loader:
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
labels = labels.type(torch.FloatTensor ).cuda()
v_pred, v_feat = model(sentences, poses, lengths, x_fm, x_numerical)
 val_predicts.append(v_pred.cpu().numpy())
 val_feats.append(v_feat.cpu().numpy())
 val_predicts = np.concatenate(val_predicts)
 val_feats = np.vstack(val_feats)
 val_loss = rmse(Y_train[valid_idx], val_predicts)
 if val_loss<min_val_loss:
min_val_loss = val_loss
oof_train_nlp[valid_idx,:] = np.hstack([val_feats, val_predicts])
 test_feats = []
test_preds = []
with torch.no_grad() :
for sentences, poses, lengths, x_fm, x_numerical in test_loader:
sentences = sentences.cuda()
poses = poses.cuda()
lengths = lengths.cuda()
x_fm = x_fm.cuda()
x_numerical = x_numerical.cuda()
v_pred, feat = model(sentences, poses, lengths, x_fm, x_numerical)
 test_preds.append(v_pred.cpu().numpy())
 test_feats.append(feat.cpu().numpy())
 test_feats = np.hstack([np.vstack(test_feats), np.concatenate(test_preds)])
 pred_test_y_k = getTestScore2(val_predicts.flatten() , tr_cdf)
 qwk = quadratic_weighted_kappa(Y_train[valid_idx], pred_test_y_k)
 print(epoch, "val loss:", val_loss, "val QWK_2 = ", qwk, "elapsed time:", time.time() -since)
 oof_test_nlp.append(test_feats)
 del model
del training_set
del validation_set
del sentences
del lengths
del x_fm
del x_numerical
del poses
gc.collect()
torch.cuda.empty_cache()
qwks.append(qwk)
 rmses.append(min_val_loss )<compute_test_metric>
lr = LogisticRegression(max_iter = 10000, penalty="l2" )
print('overall rmse: %.5f'%rmse(oof_train_nlp[:,-1], X_train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))<feature_engineering>
grid_values = {'C': [0.001,0.01,0.1,1,10,100,1000]}
model_lr = GridSearchCV(lr, param_grid=grid_values )
oof_test_nlp = np.mean(oof_test_nlp, axis=0 )<define_variables>
model_lr.fit(X_resampled, y_resampled)
 model_lr.best_params_
features = [x for x in X_train.columns if x not in to_drop_columns+svd_resnet50_features.columns.tolist() ]<train_model>
lr_1 = LogisticRegression(C = 1, max_iter = 10000, penalty="l2" )
def run_lgb(X_train, X_test, features, split_index):
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.02,
'subsample': 0.85,
'feature_fraction': 0.7,
'lambda_l1':0.01,
'verbosity': -1,
}
oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = []
qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
since = time.time()
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[valid_idx]
y_tr = X_tr['AdoptionSpeed'].values
y_val = X_val['AdoptionSpeed'].values
d_train = lgb.Dataset(X_tr[features], label=y_tr, 
)
 d_valid = lgb.Dataset(X_val[features], label=y_val, reference=d_train)
 watchlist = [d_valid]
print('training LGB:')
 lgb_model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=500,
early_stopping_rounds=100, 
)
 val_pred = lgb_model.predict(X_val[features])
 test_pred = lgb_model.predict(X_test[features])
 train_pred = lgb_model.predict(X_tr[features])
 oof_train_lgb[valid_idx] = val_pred
oof_test_lgb.append(test_pred)
 hist = histogram(X_tr['AdoptionSpeed'].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 _, cutoff = getScore(train_pred, tr_cdf, True)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(X_val['AdoptionSpeed'].values, pred_test_y_k)
 qwks.append(qwk)
 rmses.append(rmse(X_val['AdoptionSpeed'].values, val_pred))
print("QWK_2 = ", qwk,'elapsed time:',time.time() -since)
 oof_test_lgb = np.mean(oof_test_lgb, axis=0)
 print('overall rmse: %.5f'%rmse(oof_train_lgb, X_train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
return oof_train_lgb, oof_test_lgb<define_variables>
roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(lr_1, skf, X_resampled, y_resampled )
base_cols = [x for x in senti_df.columns.tolist() +meta_df.columns.tolist() +feat_df.columns.tolist() \
+cluster_img_features.columns.tolist() \
+breed_color_lda.columns.tolist() +breed_breed_lda.columns.tolist() \
+state_breed_lda.columns.tolist() \
+state_color_lda.columns.tolist() +text_features.columns.tolist() if x not in to_drop_columns]<train_model>
model_lr.fit(data_dun_train, y_train)
 model_lr.best_params_
cols34 = base_cols+['res34_%d'%IMG_FEATURE_DIM_NN2]+['resnet34_SVD_%d'%i for i in range(32)]
lgb34_oof, lgb34_test = run_lgb(X_train, X_test, cols34, split_index )<train_model>
lr_2 = LogisticRegression(C = 10, max_iter = 10000, penalty="l2" )
cols50 = base_cols+['resnet_%d'%IMG_FEATURE_DIM_NN]+['resnet50_SVD_%d'%i for i in range(32)]
lgb50_oof, lgb50_test = run_lgb(X_train, X_test, cols50, split_index )<train_model>
roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(lr, skf, data_dun_train, y_train )
cols121 = base_cols+['dense121_%d'%IMG_FEATURE_DIM_NN]+['dense121_SVD_%d'%i for i in range(32)]
lgb121_oof, lgb121_test = run_lgb(X_train, X_test, cols121, split_index )<train_model>
random_forest = RandomForestClassifier()
cols201 = base_cols+['dense_%d'%IMG_FEATURE_DIM_NN]+['dense_SVD_%d'%i for i in range(32)]
lgb201_oof, lgb201_test = run_lgb(X_train, X_test, cols201, split_index )<prepare_x_and_y>
n_estimators = [100, 300, 500, 800, 1200]
max_depth = [5, 8, 15, 25, 30]
min_samples_split = [2, 5, 10, 15, 100]
min_samples_leaf = [1, 2, 5, 10]
hyperF = dict(n_estimators = n_estimators, max_depth = max_depth,
min_samples_split = min_samples_split,
min_samples_leaf = min_samples_leaf)
 gridF = GridSearchCV(random_forest, hyperF, cv = 5, verbose = 1,
n_jobs = -1)
 bestF = gridF.fit(data_dun_train, y_train )
<compute_test_metric>
gridF.best_params_
<categorify>
random_forest = RandomForestClassifier(max_depth = 25, min_samples_leaf = 1, min_samples_split = 15, n_estimators = 500)
 roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(random_forest, skf, data_dun_train, y_train )
def mean_encode(train_data, test_data, columns, target_col, reg_method=None,
alpha=0, add_random=False, rmean=0, rstd=0.1, folds=1):

 encoded_cols = []
target_mean_global = train_data[target_col].mean()
for col in columns:
nrows_cat = train_data.groupby(col)[target_col].count()
target_means_cats = train_data.groupby(col)[target_col].mean()
target_means_cats_adj =(target_means_cats*nrows_cat +
target_mean_global*alpha)/(nrows_cat+alpha)
 encoded_col_test = test_data[col].map(target_means_cats_adj)
 if reg_method == 'expanding_mean':
train_data_shuffled = train_data.sample(frac=1, random_state=1)
 cumsum = train_data_shuffled.groupby(col)[target_col].cumsum() - train_data_shuffled[target_col]
cumcnt = train_data_shuffled.groupby(col ).cumcount()
encoded_col_train = cumsum/(cumcnt)
 encoded_col_train.fillna(target_mean_global, inplace=True)
 if add_random:
encoded_col_train = encoded_col_train + normal(loc=rmean, scale=rstd,
size=(encoded_col_train.shape[0]))
elif(reg_method == 'k_fold')and(folds > 1):
kf = StratifiedKFold(folds, shuffle=True, random_state=1)
 kfold = kf.split(train_data, train_data[target_col].values)
 parts = []
for tr_in, val_ind in kfold:
df_for_estimation, df_estimated = train_data.iloc[tr_in], train_data.iloc[val_ind]
nrows_cat = df_for_estimation.groupby(col)[target_col].count()
target_means_cats = df_for_estimation.groupby(col)[target_col].mean()
target_means_cats_adj =(target_means_cats*nrows_cat +
target_mean_global*alpha)/(nrows_cat+alpha)
 encoded_col_train_part = df_estimated[col].map(target_means_cats_adj)
 if add_random:
encoded_col_train_part = encoded_col_train_part + normal(loc=rmean, scale=rstd,
size=(encoded_col_train_part.shape[0]))
parts.append(encoded_col_train_part)
 encoded_col_train = pd.concat(parts, axis=0)
 encoded_col_train.fillna(target_mean_global, inplace=True)
 else:
encoded_col_train = train_data[col].map(target_means_cats_adj)
 if add_random:
encoded_col_train = encoded_col_train + normal(loc=rmean, scale=rstd,
size=(encoded_col_train.shape[0]))
encoded_col = pd.concat([encoded_col_train, encoded_col_test], axis=0)
 encoded_col[encoded_col.isnull() ] = target_mean_global
encoded_cols.append(pd.DataFrame({'mean_'+target_col+'_'+col:encoded_col}))
all_encoded = pd.concat(encoded_cols, axis=1)
 return(all_encoded[:train_data.shape[0]],
all_encoded[train_data.shape[0]:] )<define_variables>
params = {
"loss" : ["hinge", "log", "squared_hinge", "modified_huber"],
"alpha" : [0.0001, 0.001, 0.01, 0.1],
"penalty" : ["l2", "l1", "none"]
}
model = SGDClassifier(max_iter=5, tol=None)
 clf = GridSearchCV(model, param_grid=params )
features = [x for x in X_train.columns if x not in to_drop_columns+svd_dense_features.columns.tolist() ]<define_variables>
bestF_sgd = clf.fit(data_dun_train, y_train )
xgb_features = features<define_variables>
bestF_sgd.best_score_
params = {
'objective': 'reg:linear',
'eval_metric':'rmse',
'eta': 0.01,
'tree_method':'gpu_hist',
'max_depth': 9,
'subsample': 0.85,
'colsample_bytree': 0.7,
'alpha': 0.01,
}
oof_train_xgb = np.zeros(( X_train.shape[0]))
oof_test_xgb = []
qwks = []
i = 0
test_set = xgb.DMatrix(X_test[xgb_features])
 for n_fold,(train_idx, valid_idx)in enumerate(split_index):
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[valid_idx]
y_tr = X_tr['AdoptionSpeed'].values
y_val = X_val['AdoptionSpeed'].values
d_train = xgb.DMatrix(X_tr[xgb_features], y_tr)
 d_valid = xgb.DMatrix(X_val[xgb_features], y_val)
 watchlist = [d_valid]
since = time.time()
print('training XGB:')
 model = xgb.train(params, d_train, num_boost_round = 10000, evals=[(d_valid,'val')],
early_stopping_rounds=100,
verbose_eval=500)
 val_pred = model.predict(d_valid)
 test_pred = model.predict(test_set)
 oof_train_xgb[valid_idx] = val_pred
oof_test_xgb.append(test_pred)
 tr_cdf = get_cdf(hist)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(X_val['AdoptionSpeed'].values, pred_test_y_k)
 qwks.append(qwk)
 print("QWK_2 = ", qwk,'elapsed time:',time.time() -since )<compute_test_metric>
bestF_sgd.best_estimator_
print('overall rmse: %.5f'%rmse(oof_train_xgb, X_train['AdoptionSpeed']))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))<compute_test_metric>
sgd_best = SGDClassifier(alpha=0.001, loss='log', max_iter=5, penalty='l1', tol=None )
final_pred = pd.DataFrame()
final_pred['lgb34'] = getTestScore2(lgb34_oof, tr_cdf)
 final_pred['lgb50'] = getTestScore2(lgb50_oof, tr_cdf)
 final_pred['lgb121'] = getTestScore2(lgb121_oof, tr_cdf)
 final_pred['lgb201'] = getTestScore2(lgb201_oof, tr_cdf)
 final_pred['xgb'] = getTestScore2(oof_train_xgb, tr_cdf)
 final_pred['nlp_pred'] = getTestScore2(oof_train_nlp[:,-1], tr_cdf )<feature_engineering>
roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc_2(sgd_best, skf, data_dun_train, y_train )
final_pred['pred'] = final_pred[['lgb34','lgb50','lgb121','lgb201','xgb','nlp_pred']].mode(axis=1)[0]<compute_test_metric>
X_train, X_test, y_train_2, y_test = train_test_split(data_dun_train, y_train, test_size=0.25 )
qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, final_pred['pred'])
 print(qwk )<compute_test_metric>
knn = KNeighborsClassifier()
k_choices = [1, 2, 3, 5, 8, 10, 15, 20, 25, 50]
for k in k_choices:
knn = KNeighborsClassifier(n_neighbors = k)
 knn.fit(X_train, y_train_2)
 y_pred_prob = knn.predict_proba(X_test)[:,1]
print('k = ', k, ' ROC-AUC:', roc_auc_score(y_test, y_pred_prob))
final_pred = pd.DataFrame()
final_pred['lgb34'] = getTestScore2(lgb34_test, tr_cdf)
 final_pred['lgb50'] = getTestScore2(lgb50_test, tr_cdf)
 final_pred['lgb121'] = getTestScore2(lgb121_test, tr_cdf)
 final_pred['lgb201'] = getTestScore2(lgb201_test, tr_cdf)
 final_pred['xgb'] = getTestScore2(np.mean(oof_test_xgb,axis=0), tr_cdf)
 final_pred['nlp_pred'] = getTestScore2(oof_test_nlp[:,-1], tr_cdf)
 final_pred['pred'] = final_pred[['lgb34','lgb50','lgb121','lgb201','xgb','nlp_pred']].mode(axis=1)[0]<compute_test_metric>
knn_best = KNeighborsClassifier(n_neighbors=15)
 roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(knn_best, skf, data_dun_train, y_train )
np.corrcoef([lgb34_test,
lgb50_test,
lgb121_test,
lgb201_test,
np.mean(oof_test_xgb,axis=0),
oof_test_nlp[:,-1]] )<concatenate>
gaussian = GaussianNB()
roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(gaussian, skf, data_dun_train, y_train )
X_train_stacking = np.vstack([lgb34_oof,
lgb50_oof,
lgb121_oof,
lgb201_oof,
oof_train_xgb,
oof_train_nlp[:,-1]] ).T
X_test_stacking = np.vstack([lgb34_test,
lgb50_test,
lgb121_test,
lgb201_test,
np.mean(oof_test_xgb,axis=0),
oof_test_nlp[:,-1]] ).T
stacking_train = np.zeros(( X_train.shape[0]))
stacking_test = []
rmses, qwks = [], []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
X_tr = X_train_stacking[train_idx]
X_val = X_train_stacking[valid_idx]
y_tr = X_train.iloc[train_idx]['AdoptionSpeed'].values
y_val = X_train.iloc[valid_idx]['AdoptionSpeed'].values
since = time.time()
print('training Ridge:')
 model = Ridge(alpha=1)
 model.fit(X_tr, y_tr)
 print(model.coef_)
 val_pred = model.predict(X_val)
 test_pred = model.predict(X_test_stacking)
 stacking_train[valid_idx] = val_pred
stacking_test.append(test_pred)
 loss = rmse(Y_train[valid_idx], val_pred)
 hist = histogram(y_tr.astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 pred_test_y_k = getTestScore2(val_pred, tr_cdf)
 qwk = quadratic_weighted_kappa(y_val, pred_test_y_k)
 qwks.append(qwk)
 rmses.append(loss)
 print("RMSE=",loss, "QWK_2 = ", qwk,'elapsed time:',time.time() -since)
 stacking_test = np.mean(stacking_test, axis=0)
 print('mean rmse:',np.mean(rmses), 'rmse std:', np.std(rmses))
print('mean qwk:', np.mean(qwks), 'qwk std:', np.std(qwks))<compute_train_metric>
params = {
"max_iter" : [1, 5, 10, 50, 100, 1000],
"alpha" : [0.0001, 0.001, 0.01, 0.1],
"penalty" : ["l2", "l1", "none"]
}
model = Perceptron(tol=None)
 per = GridSearchCV(model, param_grid=params )
hist = histogram(X_train['AdoptionSpeed'].astype(int),
int(np.min(X_train['AdoptionSpeed'])) ,
int(np.max(X_train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 train_predictions = getTestScore2(stacking_train, tr_cdf)
 test_predictions = getTestScore2(stacking_test, tr_cdf)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, train_predictions)
 print("QWK = ", qwk )<save_to_csv>
per_best = per.fit(data_dun_train, y_train )
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': stacking_test.astype(np.int32)})
 submission.to_csv('submission.csv', index=False )<import_modules>
per_best.best_score_
Dropout, PReLU, LeakyReLU, Embedding
Embedding, SpatialDropout1D, concatenate, CuDNNGRU, CuDNNLSTM, Dropout, TimeDistributed, Activation
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<load_from_csv>
per_best.best_params_
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 len_train = len(train )<define_variables>
perceptron = Perceptron(alpha=0.001, max_iter=1000, penalty='l1', tol=None)
 roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc_2(perceptron, skf, data_dun_train, y_train )
HD_cate = [
'Type',
'Breed1', 'Breed2', 'Color1', 'Color2', 'Color3', 'FurLength',
'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'Quantity', 'State',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName', 'breed_id_cross'
,'breed_type_cross'
]
keep_cate = [
'Type',
'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'Quantity', 'State',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName', 'breed_id_cross'
,'breed_type_cross'
]
keep_float = [
'Age',
'MaturitySize',
'Fee',
'VideoAmt', 'PhotoAmt','sentiment_magnitude',
'sentiment_score', 'sentiment_magnitude_sum', 'sentiment_score_sum', 'sentiment_magnitude_mean', 'sentiment_score_mean',
'sentiment_magnitude_var', 'sentiment_score_var', 'metadata_annots_score_SUM', 'metadata_annots_score_MEAN', 'metadata_annots_score_VAR',
'metadata_color_score_SUM', 'metadata_color_score_MEAN', 'metadata_color_score_VAR', 'metadata_color_pixelfrac_SUM', 'metadata_color_pixelfrac_MEAN',
'metadata_color_pixelfrac_VAR', 'metadata_crop_conf_SUM', 'metadata_crop_conf_MEAN', 'metadata_crop_conf_VAR', 'metadata_crop_importance_SUM',
'metadata_crop_importance_MEAN', 'metadata_crop_importance_VAR',
'TFIDF_Description_0', 'TFIDF_Description_1', 'TFIDF_Description_2', 'TFIDF_Description_3', 'TFIDF_Description_4',
'TFIDF_Description_5', 'TFIDF_Description_6', 'TFIDF_Description_7', 'TFIDF_Description_8', 'TFIDF_Description_9',
'TFIDF_Description_10', 'TFIDF_Description_11', 'TFIDF_Description_12', 'TFIDF_Description_13', 'TFIDF_Description_14',
'TFIDF_Description_15', 'TFIDF_metadata_annots_top_desc_0', 'TFIDF_metadata_annots_top_desc_1', 'TFIDF_metadata_annots_top_desc_2',
'TFIDF_metadata_annots_top_desc_3', 'TFIDF_metadata_annots_top_desc_4', 'TFIDF_metadata_annots_top_desc_5', 'TFIDF_metadata_annots_top_desc_6',
'TFIDF_metadata_annots_top_desc_7', 'TFIDF_metadata_annots_top_desc_8', 'TFIDF_metadata_annots_top_desc_9',
'TFIDF_metadata_annots_top_desc_10', 'TFIDF_metadata_annots_top_desc_11', 'TFIDF_metadata_annots_top_desc_12',
'TFIDF_metadata_annots_top_desc_13', 'TFIDF_metadata_annots_top_desc_14', 'TFIDF_metadata_annots_top_desc_15',
'TFIDF_sentiment_entities_0', 'TFIDF_sentiment_entities_1', 'TFIDF_sentiment_entities_2', 'TFIDF_sentiment_entities_3',
'TFIDF_sentiment_entities_4', 'TFIDF_sentiment_entities_5', 'TFIDF_sentiment_entities_6', 'TFIDF_sentiment_entities_7',
'TFIDF_sentiment_entities_8', 'TFIDF_sentiment_entities_9', 'TFIDF_sentiment_entities_10', 'TFIDF_sentiment_entities_11',
'TFIDF_sentiment_entities_12', 'TFIDF_sentiment_entities_13', 'TFIDF_sentiment_entities_14', 'TFIDF_sentiment_entities_15',
'IMG_SVD_0', 'IMG_SVD_1', 'IMG_SVD_2', 'IMG_SVD_3', 'IMG_SVD_4', 'IMG_SVD_5', 'IMG_SVD_6', 'IMG_SVD_7', 'IMG_SVD_8', 'IMG_SVD_9', 'IMG_SVD_10', 'IMG_SVD_11',
'IMG_SVD_12', 'IMG_SVD_13', 'IMG_SVD_14', 'IMG_SVD_15', 'IMG_SVD_16', 'IMG_SVD_17', 'IMG_SVD_18', 'IMG_SVD_19', 'IMG_SVD_20', 'IMG_SVD_21', 'IMG_SVD_22',
'IMG_SVD_23', 'IMG_SVD_24', 'IMG_SVD_25', 'IMG_SVD_26', 'IMG_SVD_27', 'IMG_SVD_28', 'IMG_SVD_29', 'IMG_SVD_30', 'IMG_SVD_31',
'average_blue_sum', 'average_blue_mean', 'average_blue_var', 'average_green_sum', 'average_green_mean', 'average_green_var',
'average_red_sum', 'average_red_mean', 'average_red_var', 'img_average_sum', 'img_average_mean', 'img_average_var',
'blur_sum', 'blur_mean', 'blur_var', 'image_size_sum', 'image_size_mean', 'image_size_var', 'width_sum', 'width_mean',
'width_var', 'height_sum', 'height_mean', 'height_var',
'RescuerID_COUNT', 'Length_Description', 'Length_metadata_annots_top_desc', 'Lengths_sentiment_entities',
'Length_Description_2', 'Length_metadata_annots_top_desc_2', 'Lengths_sentiment_entities_2',
'resultion','ppi','txt_pred','img_pred','tablur_pred'
]<merge>
decision_tree = DecisionTreeClassifier()
def agg_helper(X, base, other):
tmp2 = X.groupby(base ).agg({other:['max','min','mean']})
 tmp2.columns = pd.Index([base +'_' + e[0] + "_" + e[1].upper() for e in tmp2.columns.tolist() ])
 X = X.merge(tmp2, on = [base],how = 'left')
 return X<import_modules>
criterion = ['gini', 'entropy']
max_depth = [4,6,8,12]
parameters = dict(criterion=['gini', 'entropy'],
max_depth = [1,2,4,6,8,12],
min_samples_split = [2, 5, 10, 15, 20, 25, 30, 35, 100],
min_samples_leaf = [1, 2, 5, 10])
 clf_dt = GridSearchCV(decision_tree, parameters)
 clf_dt.fit(data_dun_train, y_train)
 print('Best Criterion:', clf_dt.best_estimator_.get_params() ['criterion'])
 print('Best max_depth:', clf_dt.best_estimator_.get_params() ['max_depth'])
 print('Best min samples split:', clf_dt.best_estimator_.get_params() ['min_samples_split'])
 print('Best max depth:', clf_dt.best_estimator_.get_params() ['min_samples_leaf'])
 CV_Result = cross_val_score(clf_dt, data_dun_train, y_train, cv=5, n_jobs=-1)
 print() ; print(CV_Result)
 print() ; print(CV_Result.mean())
 print() ; print(CV_Result.std() )
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
clf_dt.best_params_
img_size = 256
batch_size = 256<define_search_model>
decision_tree_best = DecisionTreeClassifier(criterion='gini',max_depth=4,min_samples_leaf=1,min_samples_split=30)
 roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(decision_tree_best, skf, data_dun_train, y_train )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
gbm_param_grid = {
'n_estimators': range(8, 20),
'max_depth': range(6, 10),
'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'colsample_bytree': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'colsample_bylevel': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'learning_rate': [.4,.45,.5,.55,.6],
'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0, 10.0],
'gamma': [0, 0.25, 0.5, 1.0],
'reg_lambda': [0.1, 1.0, 5.0, 10.0, 50.0, 100.0]
}
gbm = XGBClassifier(n_estimators=10)
 xgb_random = RandomizedSearchCV(param_distributions=gbm_param_grid,
estimator = gbm, scoring = "accuracy",
verbose = 1, n_iter = 50, cv = 4)
 xgb_random.fit(data_dun_train, y_train)
 print("Best parameters found: ", xgb_random.best_params_)
 print("Best accuracy found: ", xgb_random.best_score_ )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
best_score = xgb_random.best_score_
best_params = xgb_random.best_params_
print("Best score: {}".format(best_score))
print("Best params: ")
 for param_name in sorted(best_params.keys()):
print('%s: %r' %(param_name, best_params[param_name]))
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
xgb_best = XGBClassifier(colsample_bylevel=1.0, colsample_bytree=0.7,gamma=1.0,learning_rate=0.6,
max_depth=7,min_child_weight=0.5,n_estimators=17,reg_lambda=5.0,
subsample=0.7 )
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<set_options>
roc_auc, pr_auc, precision, recall, models = get_cv_roc_pr_rc(xgb_best, skf, data_dun_train, y_train )
del m
gc.collect()<create_dataframe>
random_forest.fit(data_dun_train, y_train )
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
y_pred = pd.DataFrame(random_forest.predict(data_dun_test), columns=['Survived'] )
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
sub = pd.concat([test.PassengerId, y_pred], axis=1 )
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
sub = sub.set_index(['PassengerId'] )
features_df = pd.concat([train_feats, test_feats], axis=0 )<choose_model_class>
sub.to_csv('titanic_RF_1.csv' )
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features = features_df[[f'pic_{i}' for i in range(256)]].values<create_dataframe>
y_pred = pd.DataFrame(xgb_best.predict(data_dun_test), columns=['Survived'] )
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<concatenate>
sub = pd.concat([test.PassengerId, y_pred], axis=1 )
img_row = pd.concat([all_ids.reset_index(drop = True), features_df[[f'pic_{i}' for i in range(256)]].reset_index(drop = True)], axis=1 )<load_from_csv>
sub = sub.set_index(['PassengerId'] )
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
sub.to_csv('titanic_XGB_1.csv' )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
import keras
from keras.models import Sequential
from keras.layers import Dense
from numpy.random import seed
import tensorflow
from keras.layers import Dense, Activation, Dropout
from keras.wrappers.scikit_learn import KerasClassifier
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_max': file_sentences_sentiment['magnitude'].max(axis=0),
'score_max': file_sentences_sentiment['score'].max(axis=0),
'magnitude_min': file_sentences_sentiment['magnitude'].min(axis=0),
'score_min': file_sentences_sentiment['score'].min(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 try:
metadata_file = pet_parser.open_json_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
if dfs_metadata:
dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<data_type_conversions>
ss = StandardScaler()
data_train_scaler = ss.fit_transform(data_dun_train)
 data_test_scaler = ss.fit_transform(data_dun_test )
aggregates = ['sum', 'mean', 'var', 'min', 'max']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
model = Sequential()
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
len(data_train_scaler[1] )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
model.add(Dense(512, activation = 'relu', input_dim = 73))
model.add(Dense(512, activation = 'relu'))
model.add(Dense(128, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<data_type_conversions>
model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'] )
X['breed_type_cross'] = X['main_breed_BreedName'].fillna('none')+'_' + X['second_breed_BreedName'].fillna('none')
 X['breed_id_cross'] = X['main_breed_Type'].fillna('none' ).astype(str)+'_' + X['second_breed_Type'].fillna('none' ).astype(str )<define_variables>
model.fit(data_train_scaler, y_train, batch_size = 10, epochs = 100 )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName','breed_id_cross','breed_type_cross']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<feature_engineering>
y_pred = pd.DataFrame(model.predict(data_test_scaler), columns=['Survived'] )
def no_name(x):
if x == 'n' or 'No_name' in x:
return 1
return 0
X_temp['Name'] = X_temp['Name'].fillna('n')
 X_temp['Name'] = X_temp['Name'].apply(lambda x:no_name(str(x)))
 X_temp['Name_length'] = X_temp['Name'].apply(lambda x:len(str(x)))
 X_temp['Name_length_split'] = X_temp['Name'].apply(lambda x:len(str(x ).split()))
def have_num(x):
if any(char.isdigit() for char in x):
return 1
else:
return 0
X_temp['Name_length_if_num'] = X_temp['Name'].apply(lambda x:have_num(str(x)))
 def kitty(x):
if 'kitty' in x.lower() :
return 1
else:
return 0
X_temp['Name_if_kitty'] = X_temp['Name'].apply(lambda x:kitty(str(x)))
 def puppy(x):
if 'puppy' in x.lower() :
return 1
else:
return 0
X_temp['Name_if_puppy'] = X_temp['Name'].apply(lambda x:puppy(str(x)))
<merge>
sub = pd.concat([test.PassengerId, y_pred], axis=1 )
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
sub['Survived'].values[sub['Survived'].values > 0.5] = 1
sub['Survived'].values[sub['Survived'].values <= 0.5] = 0
sub['Survived'] = sub['Survived'].astype(int )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<feature_engineering>
sub = sub.set_index(['PassengerId'] )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
sub.to_csv('titanic_keras_2.csv' )
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len)
 X_TXT = X_text[['Description', 'metadata_annots_top_desc', 'sentiment_entities']].copy()<feature_engineering>
tmodel = Sequential()
tmodel.add(Dense(input_dim=data_train_scaler.shape[1], units=128,
kernel_initializer='normal', bias_initializer='zeros'))
tmodel.add(Activation('relu'))
for i in range(0, 8):
tmodel.add(Dense(units=64, kernel_initializer='normal',
bias_initializer='zeros'))
tmodel.add(Activation('relu'))
tmodel.add(Dropout (.25))
tmodel.add(Dense(units=1))
tmodel.add(Activation('linear'))
tmodel.compile(loss='mean_squared_error', optimizer='rmsprop' )
X_temp['Length_Description_2'] = X_text['Description'].apply(lambda x: len(x.split()))
X_temp['Length_metadata_annots_top_desc_2'] = X_text['metadata_annots_top_desc'].apply(lambda x: len(x.split()))
X_temp['Lengths_sentiment_entities_2'] = X_text['sentiment_entities'].apply(lambda x: len(x.split())) 
<string_transform>
tmodel.fit(data_train_scaler, y_train, epochs=600, verbose=2 )
def spacing_punctuation(text):

 regular_punct = list(string.punctuation)
 extra_punct = [
',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤']
all_punct = ''.join(sorted(list(set(regular_punct + extra_punct))))
re_tok = re.compile(f'([{all_punct}])')
 return re_tok.sub(r' \1 ', text)
 def tokenize(df_text, max_features):
df_text = df_text.apply(spacing_punctuation)
 tokenizer = Tokenizer(
 num_words=max_features,
filters='',
lower=False,
split=' ')
 tokenizer.fit_on_texts(list(df_text))
return tokenizer<concatenate>
y_pred = pd.DataFrame(tmodel.predict(data_test_scaler), columns=['Survived'] )
df_text = pd.concat([X_TXT['Description'], X_TXT['metadata_annots_top_desc'],X_TXT['sentiment_entities']],axis=0 ).reset_index(drop=True)
 tokenizer = tokenize(df_text, max_features=160000 )<train_model>
sub = pd.concat([test.PassengerId, y_pred], axis=1 )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.005,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test<categorify>
sub['Survived'].values[sub['Survived'].values > 0.5] = 1
sub['Survived'].values[sub['Survived'].values <= 0.5] = 0
sub['Survived'] = sub['Survived'].astype(int )
tabular_temp = X_temp[[i for i in X_temp.columns if i not in to_drop_columns +['Description','metadata_annots_top_desc', 'sentiment_entities'] ] ]
for i in HD_cate:
if i in tabular_temp.columns:
tabular_temp[i] = tabular_temp.groupby(i)[i].transform('count')
 X_tabular_temp = tabular_temp[len_train:].drop(['AdoptionSpeed'], axis = 1 )<feature_engineering>
sub = sub.set_index(['PassengerId'] )
tablur_train, tablur_test = run_xgb(xgb_params,tabular_temp[:len_train], X_tabular_temp)
 X_temp['tablur_pred'] = np.concatenate([tablur_train,tablur_test.mean(axis = 1)] )<set_options>
sub.to_csv('titanic_keras_exp_1.csv' )
del tabular_temp, X_tabular_temp
gc.collect()<feature_engineering>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['Survived'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )
print('Tfidf word vector')
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 3),
max_features=20000)
 word_vectorizer.fit(df_text)
 train_desc_word_features = word_vectorizer.transform(X_TXT[:len_train]['Description'])
 test_desc_word_features = word_vectorizer.transform(X_TXT[len_train:]['Description'])
 train_meta_word_features = word_vectorizer.transform(X_TXT[:len_train]['metadata_annots_top_desc'])
 test_meta_word_features = word_vectorizer.transform(X_TXT[len_train:]['metadata_annots_top_desc'])
 train_sent_word_features = word_vectorizer.transform(X_TXT[:len_train]['sentiment_entities'])
 test_sent_word_features = word_vectorizer.transform(X_TXT[len_train:]['sentiment_entities'])
 print('Tfidf char vector')
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 5),
max_features=50000)
 char_vectorizer.fit(df_text)
 train_desc_char_features = char_vectorizer.transform(X_TXT[:len_train]['Description'])
 test_desc_char_features = char_vectorizer.transform(X_TXT[len_train:]['Description'])
 train_meta_char_features = char_vectorizer.transform(X_TXT[:len_train]['metadata_annots_top_desc'])
 test_meta_char_features = char_vectorizer.transform(X_TXT[len_train:]['metadata_annots_top_desc'])
 train_sent_char_features = char_vectorizer.transform(X_TXT[:len_train]['sentiment_entities'])
 test_sent_char_features = char_vectorizer.transform(X_TXT[len_train:]['sentiment_entities'])
 train_features = hstack([
train_desc_word_features, train_desc_char_features,
train_meta_word_features, train_meta_char_features,
train_sent_word_features, train_sent_char_features, ] ).tocsr()
test_features = hstack([
test_desc_word_features, test_desc_char_features,
test_meta_word_features, test_meta_char_features,
test_sent_word_features, test_sent_char_features, ] ).tocsr()
def run_txt(X_train, X_test):
n_splits = 10
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(train_features, X_train['AdoptionSpeed'].values):
X_tr,X_val= train_features[train_idx], train_features[valid_idx]
y_tr = X_train.iloc[train_idx, :]['AdoptionSpeed'].values
y_val = X_train.iloc[valid_idx, :]['AdoptionSpeed'].values
model = Ridge(alpha=20, copy_X=True, fit_intercept=True, solver='auto',max_iter=100,normalize=False, random_state=0, tol=0.0025)
 model.fit(X_tr,y_tr)
 valid_pred = model.predict(X_val)
 test_pred = model.predict(test_features)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
oof_test = oof_test.mean(1)
 return oof_train, oof_test<feature_engineering>
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )
oof_train_txt, oof_test_txt = run_txt(X_temp[:len_train][['AdoptionSpeed']], X_temp[len_train:][['AdoptionSpeed']])
 X_temp['txt_pred'] = np.concatenate([oof_train_txt,oof_test_txt] )<drop_column>
df_all['Age'] = pd.qcut(df_all['Age'], 10 )
del train_features, test_features, train_desc_word_features,test_desc_char_features,train_meta_char_features,test_meta_char_features
del train_sent_char_features,test_desc_word_features, test_meta_word_features, test_sent_word_features, test_sent_char_features,
gc.collect()<categorify>
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )
def load_word_embedding(filepath):

 def _get_vec(word, *arr):
return word, np.asarray(arr, dtype='float32')
 print('load word embedding...... ')
 try:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))
except UnicodeDecodeError:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(
 filepath, encoding="utf8", errors='ignore'))
words_to_del = []
for word, vec in word_embedding.items() :
if len(vec)!= 300:
words_to_del.append(word)
 for word in words_to_del:
del word_embedding[word]
return word_embedding<categorify>
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1
def create_embedding_weights(word_index, word_embedding,
max_features, paragram=False):
print('create word embedding weights...... ')
 count = 0
stemmer = PorterStemmer()
mat_embedding = np.stack(word_embedding.values())
 a, b = min(max_features, len(word_index)) + 1, mat_embedding.shape[1]
print('embedding weights matrix with shape:({}, {})'.format(a, b))
embedding_mean, embedding_std = mat_embedding.mean() , mat_embedding.std()
del mat_embedding
gc.collect()
embedding_weights = np.random.normal(embedding_mean, embedding_std,(a, b))
for word, idx in word_index.items() :
if idx >= a:
continue
word_vec = word_embedding.get(word, None)
 if word_vec is None:
word_vec = word_embedding.get(word.lower() , None)
 if word_vec is None:
word_vec = word_embedding.get(stemmer.stem(word), None)
 if word_vec is not None:
count += 1
if word_vec is not None:
embedding_weights[idx] = word_vec
print("{} number has been replaced".format(count))
return embedding_weights<load_pretrained>
import string
GLOVE_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
glove_word_embed = load_word_embedding(GLOVE_PATH )<load_pretrained>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]
embedding_matrix = create_embedding_weights(tokenizer.word_index, glove_word_embed, 160000, False )<set_options>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA