Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_output>
%matplotlib inline
seed=45
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore', category=DeprecationWarning)
parameters_df=pd.DataFrame()
i = 0
for breed in catdog_ratings.keys() :
for key in catdog_ratings[breed].keys() :
parameters_df.at[i,'breed'] = breed
parameters_df.at[i,key] = catdog_ratings[breed][key]
i = i+1<rename_columns>
train=pd.read_csv(".. /input/train.csv",sep=',')
 test=pd.read_csv(".. /input/test.csv",sep=',' )
parameters_df.rename(index=str, columns={'breed':'Breed1Name'},inplace=True )<merge>
train.isnull().sum()
data_df = data_df.merge(parameters_df[['Breed1Name','Affectionate with Family','Amount of Shedding',
'Easy to Groom','General Health','Intelligence','Kid Friendly',
'Pet Friendly','Potential for Playfulness']], on='Breed1Name', how='left' )<feature_engineering>
train.isnull().sum()
data_df['Description'] = data_df['Description'].fillna(' ' )<feature_engineering>
test.isnull().sum()
english_desc, chinese_desc = [], []
tokens = set()
word_dict = {}
pos_count, word_count = 1, 1
pos_dict = {}
eng_sequences = []
pos_sequences = []
for i in range(len(data_df)) :
e_d, c_d, eng_seq, pos_seq = [], [], [], []
doc = custom_tokenizer(data_df['Description'].iloc[i])
 for token in doc:
if not isascii(token):
c_d.append(token)
 else:
e_d.append(token)
 if token not in word_dict:
word_dict[token] = word_count
word_count +=1
english_desc.append(' '.join(e_d))
chinese_desc.append(' '.join(c_d))
pos_seq = nltk.pos_tag(e_d)
 for t in pos_seq:
if t[1] not in pos_dict:
pos_dict[t[1]] = pos_count
pos_count += 1
pos_seq = [pos_dict[t[1]] for t in pos_seq]
eng_seq = [word_dict[t] for t in e_d]
if len(eng_seq)==0:
eng_seq.append(0)
 pos_seq.append(0)
 eng_sequences.append(eng_seq)
 pos_sequences.append(pos_seq )<feature_engineering>
test.isnull().sum()
data_df['English_desc'] = english_desc
data_df['Chinese_desc'] = chinese_desc
data_df['e_description_len'] = data_df['English_desc'].apply(lambda x:len(x))
data_df['e_description_word_len'] = data_df['English_desc'].apply(lambda x: len(x.split(' ')))
 data_df['e_description_word_unique'] = data_df['English_desc'].apply(lambda x: len(set(x.split(' '))))
data_df['c_description_len'] = data_df['Chinese_desc'].apply(lambda x:len(x))
data_df['c_description_word_len'] = data_df['Chinese_desc'].apply(lambda x:len(x.split(' ')))
 data_df['c_description_word_unique'] = data_df['Chinese_desc'].apply(lambda x: len(set(x)))
 data_df['description_len'] = data_df['Description'].apply(lambda x:len(x))
data_df['description_word_len'] = data_df['Description'].apply(lambda x: len(x.split(' ')) )<features_selection>
train['Embarked'].value_counts()
def load_glove() :
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in(open(EMBEDDING_FILE)))
 return embeddings_index
glove_emb = load_glove()
embedding_matrix, nb_words, nb_oov = build_emb_matrix(word_dict, glove_emb)
 print(nb_words, nb_oov)
 del glove_emb
gc.collect()<define_variables>
train[train.Embarked.isnull() ]
len(set(train.index.tolist()))<define_variables>
train.Embarked.fillna("C", inplace=True )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)))
 image_files = train_image_files+test_image_files
metadata_files = train_metadata_files+test_metadata_files
sentiment_files = train_sentiment_files+test_sentiment_files<feature_engineering>
train.isnull().sum()
IMG_SIZE = 256
BATCH_SIZE = 256
def get_petid(path):
basename = os.path.basename(path)
 return basename.split('-')[0]
def get_picid(path):
basename = os.path.splitext(os.path.basename(path)) [0]
return basename.split('-')[1]
image_df = pd.DataFrame(image_files, columns=['path'])
 image_df['PetID'] = image_df['path'].apply(get_petid)
 image_df['PicID'] = image_df['path'].apply(get_picid)
 gc.collect()<load_pretrained>
train.isnull().sum()
pet_image_dataset = PetDataset(image_df)
 pet_image_loader = DataLoader(pet_image_dataset, batch_size=BATCH_SIZE, shuffle=False,
num_workers=1, pin_memory=True )<find_best_params>
test.Fare.fillna(test['Fare'].mean() ,inplace=True )
resnet34_feature = []
model = ResnetModel(resnet_fun=resnet34)
 model.init_resnet('.. /input/pretrained-pytorch-dog-and-cat-models/resnet34.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 resnet34_feature.append(y_pred.cpu().numpy())
 resnet34_feature = np.vstack(resnet34_feature)
 del model
gc.collect()
torch.cuda.empty_cache()
densenet121_feature = []
model = DenseModel(dense_func=densenet121)
 model.init_densenet('.. /input/pretrained-pytorch-dog-and-cat-models/densenet121.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 densenet121_feature.append(y_pred.cpu().numpy())
 densenet121_feature = np.vstack(densenet121_feature)
 del model
gc.collect()
torch.cuda.empty_cache()<train_on_grid>
test.Fare.isnull().sum()
densenet_feature = []
model = DenseModel()
model.init_densenet('.. /input/pytorch-pretrained-image-models/densenet201.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 densenet_feature.append(y_pred.cpu().numpy())
 densenet_feature = np.vstack(densenet_feature)
 del model
gc.collect()
torch.cuda.empty_cache()
resnet50_feature = []
model = ResnetModel()
model.init_resnet('.. /input/pytorch-pretrained-image-models/resnet50.pth')
 model.cuda()
model.eval()
with torch.no_grad() :
for img_batch in tqdm(pet_image_loader):
img_batch = img_batch.float().cuda()
y_pred = model(img_batch)
 resnet50_feature.append(y_pred.cpu().numpy())
 resnet50_feature = np.vstack(resnet50_feature )<drop_column>
a=train.groupby('Sex')['Survived'].value_counts()
a
del img_batch<set_options>
combined = train.append(test)
 combined.reset_index(inplace=True)
 combined.drop(['index', 'PassengerId'], inplace=True, axis=1 )
del model
del pet_image_loader
del pet_image_dataset
gc.collect()
torch.cuda.empty_cache()<define_variables>
combined_title = [i.split(",")[1].split(".")[0].strip() for i in combined["Name"]]
combined["Title"] = pd.Series(combined_title)
 combined["Title"].head()
RES50_IMG_FEATURE_DIM = resnet50_feature.shape[1]
DENSE_IMG_FEATURE_DIM = densenet_feature.shape[1]
RES34_IMG_FEATURE_DIM = resnet34_feature.shape[1]
DENSE121_IMG_FEATURE_DIM = densenet121_feature.shape[1]
print(RES50_IMG_FEATURE_DIM)
 print(DENSE_IMG_FEATURE_DIM)
 print(RES34_IMG_FEATURE_DIM)
 print(DENSE121_IMG_FEATURE_DIM )<create_dataframe>
combined["Title"] = combined["Title"].replace(['Capt', 'Col', 'Dr', 'Major', 'Rev'], 'officer')
 combined["Title"] = combined["Title"].replace(['Lady', 'the Countess','Countess', 'Sir', 'Jonkheer', 'Dona','Don'], 'Royalty')
 combined["Title"] = combined["Title"].replace(['Miss','Mlle'], 'Miss')
 combined["Title"] = combined["Title"].replace(['Mrs','Ms','Mme'], 'Mrs')
 combined["Title"] = combined["Title"].map({"Master":0, "Miss":1, "Mrs":2, "Mr":3, "officer":4,"Royalty":5})
 combined["Title"] = combined["Title"].astype(int )
resnet50_feature_df = pd.DataFrame(resnet50_feature, dtype=np.float32,
columns=['resnet50_%d'%i for i in range(RES50_IMG_FEATURE_DIM)])
 resnet50_feature_df['PetID'] = image_df['PetID'].values
resnet50_feature_df['PicID'] = image_df['PicID'].values
resnet50_feature_df_avg = resnet50_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
resnet50_feature_df_avg.columns = ['PetID']+['resnet50_mean_%d'%i for i in range(RES50_IMG_FEATURE_DIM)]
resnet50_feature_df_1 = resnet50_feature_df[resnet50_feature_df['PicID']=='1'].drop('PicID', axis=1)
 resnet50_feature_df = resnet50_feature_df_1.merge(resnet50_feature_df_avg, on='PetID', how='outer')
 resnet50_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(resnet50_feature_df, on='PetID', how='left')
 for c in resnet50_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
resnet50_feature_df[c] = resnet50_feature_df[c].fillna(-1)
 del resnet50_feature_df_avg
del resnet50_feature_df_1
gc.collect()<create_dataframe>
combined['family_size'] = combined.SibSp + combined.Parch+1
dense_feature_df = pd.DataFrame(densenet_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(DENSE_IMG_FEATURE_DIM)])
 dense_feature_df['PetID'] = image_df['PetID'].values
dense_feature_df['PicID'] = image_df['PicID'].values
dense_feature_df_avg = dense_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
dense_feature_df_avg.columns = ['PetID']+['densenet_mean_%d'%i for i in range(DENSE_IMG_FEATURE_DIM)]
dense_feature_df_1 = dense_feature_df[dense_feature_df['PicID']=='1'].drop('PicID', axis=1)
 dense_feature_df = dense_feature_df_1.merge(dense_feature_df_avg, on='PetID', how='left')
 dense_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(dense_feature_df, on='PetID', how='left')
 for c in dense_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
dense_feature_df[c] = dense_feature_df[c].fillna(-1)
 del dense_feature_df_1
del dense_feature_df_avg
gc.collect()<create_dataframe>
combined['Single'] = combined['family_size'].map(lambda s: 1 if s == 1 else 0)
 combined['SmallF'] = combined['family_size'].map(lambda s: 1 if s == 2 else 0)
 combined['MedF'] = combined['family_size'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 combined['LargeF'] = combined['family_size'].map(lambda s: 1 if s >= 5 else 0 )
res34_feature_df = pd.DataFrame(resnet34_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(RES34_IMG_FEATURE_DIM)])
 res34_feature_df['PetID'] = image_df['PetID'].values
res34_feature_df['PicID'] = image_df['PicID'].values
res34_feature_df_avg = res34_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
res34_feature_df_avg.columns = ['PetID']+['res34_mean_%d'%i for i in range(RES34_IMG_FEATURE_DIM)]
res34_feature_df_1 = res34_feature_df[res34_feature_df['PicID']=='1'].drop('PicID', axis=1)
 res34_feature_df = res34_feature_df_1.merge(res34_feature_df_avg, on='PetID', how='left')
 res34_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(res34_feature_df, on='PetID', how='left')
 for c in res34_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
res34_feature_df[c] = res34_feature_df[c].fillna(-1)
 del res34_feature_df_1
del res34_feature_df_avg
gc.collect()<create_dataframe>
combined['Sex'] = combined['Sex'].map({'female': 1, 'male': 0} ).astype(int )
dense121_feature_df = pd.DataFrame(densenet121_feature, dtype=np.float32,
columns=['densenet_%d'%i for i in range(DENSE121_IMG_FEATURE_DIM)])
 dense121_feature_df['PetID'] = image_df['PetID'].values
dense121_feature_df['PicID'] = image_df['PicID'].values
dense121_feature_df_avg = dense121_feature_df.drop('PicID', axis=1 ).groupby('PetID' ).agg('mean' ).reset_index()
dense121_feature_df_avg.columns = ['PetID']+['dense121_mean_%d'%i for i in range(DENSE121_IMG_FEATURE_DIM)]
dense121_feature_df_1 = dense121_feature_df[dense121_feature_df['PicID']=='1'].drop('PicID', axis=1)
 dense121_feature_df = dense121_feature_df_1.merge(dense121_feature_df_avg, on='PetID', how='left')
 dense121_feature_df = data_df[['PetID','RescuerID','AdoptionSpeed']].merge(dense121_feature_df, on='PetID', how='left')
 for c in dense121_feature_df.columns:
if c in ['PetID','RescuerID','AdoptionSpeed']:continue
dense121_feature_df[c] = dense121_feature_df[c].fillna(-1)
 del dense121_feature_df_1
del dense121_feature_df_avg
gc.collect()<choose_model_class>
combined['Embarked'] = combined['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
 combined.head()
class ImgExtractor(nn.Module):
def __init__(self, raw_feature_dim = 4096, feature_dim=64):
super(ImgExtractor, self ).__init__()
self.extractor = nn.Sequential(
 nn.BatchNorm1d(raw_feature_dim),
nn.Dropout(0.5),
nn.Linear(raw_feature_dim, raw_feature_dim//4),
nn.ELU(inplace=True),
nn.Dropout(0.1),
nn.Linear(raw_feature_dim//4, raw_feature_dim//8),
nn.ELU(inplace=True),
nn.Linear(raw_feature_dim//8, feature_dim),
nn.ELU(inplace=True)
)
 self.logit = nn.Sequential(
 nn.Linear(feature_dim, 1)
)
 def forward(self, x):
batch_size = x.shape[0]
feat = self.extractor(x)
 out = self.logit(feat)
 return out, feat<prepare_x_and_y>
Embarked_dummies = pd.get_dummies(combined['Embarked'], prefix='Embarked')
 combined = pd.concat([combined, Embarked_dummies], axis=1)
 combined.drop('Embarked', inplace=True, axis=1 )
IMG_FEATURE_DIM_NN = 128
IMG_FEATURE_DIM_NN2 = 64
def get_image_feature(img_train_data, img_test_data,img_feature_dim):
loss_fn = torch.nn.MSELoss().cuda()
oof_train_img = np.zeros(( img_train_data.shape[0], img_feature_dim+1))
oof_test_img = []
test_dataset = TensorDataset(torch.tensor(img_test_data))
test_loader = DataLoader(test_dataset, batch_size=1024, shuffle=False, num_workers=1, pin_memory=True)
 qwks = []
rmses = []
for n_fold,(train_idx, valid_idx)in enumerate(split_index):
print('fold:',n_fold)
 X_tr = img_train_data[train_idx]
X_val = img_train_data[valid_idx]
y_tr = train.iloc[train_idx]['AdoptionSpeed'].values
y_val = train.iloc[valid_idx]['AdoptionSpeed'].values
hist = histogram(y_tr.astype(int),
int(np.min(train['AdoptionSpeed'])) ,
int(np.max(train['AdoptionSpeed'])))
 tr_cdf = get_cdf(hist)
 tra_dataset = TensorDataset(torch.tensor(X_tr), torch.tensor(y_tr))
val_dataset = TensorDataset(torch.tensor(X_val), torch.tensor(y_val))
training_loader = DataLoader(tra_dataset, batch_size=1024, shuffle=True, num_workers=1, pin_memory=True)
 validation_loader = DataLoader(val_dataset, batch_size=1024, shuffle=False, num_workers=1, pin_memory=True)
 model = ImgExtractor(raw_feature_dim=img_train_data.shape[1], feature_dim=img_feature_dim)
 model.cuda()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.002)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=7, eta_min=0.0003)
 iteration = 0
min_val_loss = 100
since = time.time()
test_feats = None
for epoch in range(15):
scheduler.step()
model.train()
for x, y in training_loader:
iteration += 1
x = x.cuda()
y = y.type(torch.FloatTensor ).cuda().view(-1, 1)
 pred, feat = model(x)
 loss = loss_fn(pred, y)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
val_predicts = []
val_feats = []
with torch.no_grad() :
for x, y in validation_loader:
x = x.cuda()
y = y.type(torch.FloatTensor ).cuda()
v_pred, feat = model(x)
 val_predicts.append(v_pred.cpu().numpy())
 val_feats.append(feat.cpu().numpy())
 val_predicts = np.concatenate(val_predicts)
 val_feats = np.vstack(val_feats)
 pred_test_y_k = getTestScore2(val_predicts.flatten() , tr_cdf)
 qwk = quadratic_weighted_kappa(y_val, pred_test_y_k)
 val_loss = rmse(y_val,val_predicts)
 if val_loss<min_val_loss:
min_val_loss = val_loss
oof_train_img[valid_idx,:] = np.hstack([val_feats,val_predicts])
 test_feats = []
test_preds = []
with torch.no_grad() :
for x, in test_loader:
x = x.cuda()
v_pred, feat = model(x)
 test_preds.append(v_pred.cpu().numpy())
 test_feats.append(feat.cpu().numpy())
 test_feats = np.hstack([np.vstack(test_feats), np.concatenate(test_preds)])
 print(epoch, "best loss! val loss:", val_loss, 'qwk:', qwk, "elapsed time:", time.time() -since)
 oof_test_img.append(test_feats)
 rmses.append(min_val_loss)
 qwks.append(qwk)
 del model
del x
del y
del tra_dataset
del val_dataset
del training_loader
del validation_loader
gc.collect()
torch.cuda.empty_cache()
print('overall rmse: %.5f'%rmse(oof_train_img[:,-1], train['AdoptionSpeed']))
print('mean rmse =', np.mean(rmses), 'rmse std =', np.std(rmses))
print('mean QWK =', np.mean(qwks), 'std QWK =', np.std(qwks))
del test_loader
del test_dataset
gc.collect()
return oof_train_img, np.mean(oof_test_img, axis=0 )<prepare_x_and_y>
front = train['Age']
train.drop(labels=['Age'], axis=1,inplace = True)
 train.insert(0, 'Age', front)
 train.head()
res34_array = res34_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
res34_tra = res34_array[0:train_len]
res34_test = res34_array[train_len:]
oof_train_res34, oof_test_res34 = get_image_feature(res34_tra, res34_test, IMG_FEATURE_DIM_NN2)
 del res34_tra
del res34_test
gc.collect()
dnn_resnet34_features = np.vstack([oof_train_res34, oof_test_res34])
 dnn_resnet34_features = pd.DataFrame(dnn_resnet34_features, columns=['res34_%d'%c for c in range(dnn_resnet34_features.shape[1])])
 dnn_resnet34_features['PetID'] = data_df['PetID'].values
del oof_train_res34
del oof_test_res34
gc.collect()<normalization>
front = test['Age']
test.drop(labels=['Age'], axis=1,inplace = True)
 test.insert(0, 'Age', front)
 test.head()
stdscaler = RobustScaler()
res50_array = resnet50_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
res50_tra = res50_array[0:train_len]
res50_test = res50_array[train_len:]
oof_train_res, oof_test_res = get_image_feature(res50_tra, res50_test, IMG_FEATURE_DIM_NN)
 del res50_tra
del res50_test
gc.collect()
dnn_resnet50_features = np.vstack([oof_train_res, oof_test_res])
 dnn_resnet50_features = pd.DataFrame(dnn_resnet50_features, columns=['resnet_%d'%c for c in range(dnn_resnet50_features.shape[1])])
 dnn_resnet50_features['PetID'] = data_df['PetID'].values
del oof_train_res
del oof_test_res
gc.collect()<feature_engineering>
guess_ages = np.zeros(( 2,3))
guess_ages
dense121_array = dense121_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
dense121_tra = dense121_array[0:train_len]
dense121_test = dense121_array[train_len:]
oof_train_dense121, oof_test_dense121 = get_image_feature(dense121_tra, dense121_test, IMG_FEATURE_DIM_NN)
 del dense121_tra
del dense121_test
gc.collect()
dnn_dense121_features = np.vstack([oof_train_dense121, oof_test_dense121])
 dnn_dense121_features = pd.DataFrame(dnn_dense121_features, columns=['dense121_%d'%c for c in range(dnn_dense121_features.shape[1])])
 dnn_dense121_features['PetID'] = data_df['PetID'].values
del oof_train_dense121
del oof_test_dense121
gc.collect()<feature_engineering>
for i in range(0, 2):
for j in range(0, 3):
guess_df = combined[(combined['Sex'] == i)& 
(combined['Pclass'] == j+1)]['Age'].dropna().astype(int)
 age_guess = guess_df.median()
guess_ages[i,j] =int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
combined.loc[(combined.Age.isnull())&(combined.Sex == i)&(combined.Pclass == j+1),\
'Age'] = guess_ages[i,j]
combined['Age'] = combined['Age'].astype(int)
 train.head()
dense_array = dense_feature_df.drop(['PetID','RescuerID','AdoptionSpeed'], axis=1 ).values
dense_tra = dense_array[0:train_len]
dense_test = dense_array[train_len:]
oof_train_dense, oof_test_dense = get_image_feature(dense_tra, dense_test, IMG_FEATURE_DIM_NN)
 del dense_tra
del dense_test
gc.collect()
dnn_dense_features = np.vstack([oof_train_dense, oof_test_dense])
 dnn_dense_features = pd.DataFrame(dnn_dense_features, columns=['dense_%d'%c for c in range(dnn_dense_features.shape[1])])
 dnn_dense_features['PetID'] = data_df['PetID'].values
del oof_train_dense
del oof_test_dense
gc.collect()<feature_engineering>
combined.isnull().sum()
n_components = 64
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_resnet50_features = svd_.fit_transform(res50_array)
 svd_resnet50_features = pd.DataFrame(svd_resnet50_features)
 svd_resnet50_features = svd_resnet50_features.add_prefix('resnet50_SVD_')
 svd_resnet50_features['PetID'] = resnet50_feature_df['PetID'].values<feature_engineering>
train['AgeBand'] = pd.cut(train['Age'], 5)
 train[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_resnet34_features = svd_.fit_transform(res34_array)
 svd_resnet34_features = pd.DataFrame(svd_resnet34_features)
 svd_resnet34_features = svd_resnet34_features.add_prefix('resnet34_SVD_')
 svd_resnet34_features['PetID'] = resnet50_feature_df['PetID'].values<feature_engineering>
combined.loc[ combined['Age'] <= 16, 'Age'] = 0
combined.loc[(combined['Age'] > 16)&(combined['Age'] <= 32), 'Age'] = 1
combined.loc[(combined['Age'] > 32)&(combined['Age'] <= 48), 'Age'] = 2
combined.loc[(combined['Age'] > 48)&(combined['Age'] <= 64), 'Age'] = 3
combined.loc[ combined['Age'] > 64]
combined.head()
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_dense121_features = svd_.fit_transform(dense121_array)
 svd_dense121_features = pd.DataFrame(svd_dense121_features)
 svd_dense121_features = svd_dense121_features.add_prefix('dense121_SVD_')
 svd_dense121_features['PetID'] = resnet50_feature_df['PetID'].values<create_dataframe>
train.drop(['AgeBand'], axis=1)
 combined.head()
svd_ = TruncatedSVD(n_components=32, random_state=1337)
 svd_dense_features = svd_.fit_transform(dense_array)
 svd_dense_features = pd.DataFrame(svd_dense_features)
 svd_dense_features = svd_dense_features.add_prefix('dense_SVD_')
 svd_dense_features['PetID'] = dense_feature_df['PetID'].values<feature_engineering>
combined["Cabin"].isnull().sum()
svd_ = TruncatedSVD(n_components=64, random_state=1337)
 svd_img_features = svd_.fit_transform(np.hstack([res50_array, dense_array, dense121_array, res34_array]))
svd_img_features = pd.DataFrame(svd_img_features)
 svd_img_features = svd_img_features.add_prefix('IMG_SVD_')
 svd_img_features['PetID'] = resnet50_feature_df['PetID'].values<feature_engineering>
combined["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in combined['Cabin'] ] )
features = svd_img_features.drop(['PetID'], axis=1 ).values
cluster = KMeans(n_clusters=32)
 cluster_label = cluster.fit_predict(features)
 cluster_img_features = pd.DataFrame(cluster_label)
 cluster_img_features = cluster_img_features.add_prefix('img_CLUSTER_')
 cluster_img_features['PetID'] = dense_feature_df['PetID'].values 
<load_pretrained>
cabin_dummies = pd.get_dummies(combined['Cabin'], prefix='Cabin')
 combined = pd.concat([combined, cabin_dummies], axis=1)
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = '; '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 ret_val = {}
ret_val['doc_mag'] = file['documentSentiment']['magnitude']
ret_val['doc_score']= file['documentSentiment']['score']
ret_val['doc_language'] = file['language']
ret_val['doc_stcs_len'] = len(file['sentences'])
 if ret_val['doc_stcs_len']>0:
ret_val['doc_first_score'] = file['sentences'][0]['sentiment']['score']
ret_val['doc_first_mag'] = file['sentences'][0]['sentiment']['magnitude']
ret_val['doc_last_score'] = file['sentences'][-1]['sentiment']['score']
ret_val['doc_last_mag'] = file['sentences'][-1]['sentiment']['magnitude']
else:
ret_val['doc_first_score'] = np.nan
ret_val['doc_first_mag'] = np.nan
ret_val['doc_last_score'] = np.nan
ret_val['doc_last_mag'] = np.nan
ret_val['doc_ent_num'] = len(file['entities'])
 mags, scores = [], []
for s in file['sentences']:
mags.append(s['sentiment']['magnitude'])
 scores.append(s['sentiment']['score'])
 if len(scores)==0:
ret_val['doc_score_sum'] = np.nan
ret_val['doc_mag_sum'] = np.nan
ret_val['doc_score_mena'] = np.nan
ret_val['doc_mag_mean'] = np.nan
ret_val['doc_score_max'] = np.nan
ret_val['doc_mag_max'] = np.nan
ret_val['doc_score_min'] = np.nan
ret_val['doc_mag_min'] = np.nan
ret_val['doc_score_std'] = np.nan
ret_val['doc_mag_std'] = np.nan
else:
ret_val['doc_score_sum'] = np.sum(scores)
 ret_val['doc_mag_sum'] = np.sum(mags)
 ret_val['doc_score_mena'] = np.mean(scores)
 ret_val['doc_mag_mean'] = np.mean(mags)
 ret_val['doc_score_max'] = np.max(scores)
 ret_val['doc_mag_max'] = np.max(mags)
 ret_val['doc_score_min'] = np.min(scores)
 ret_val['doc_mag_min'] = np.min(mags)
 ret_val['doc_score_std'] = np.std(scores)
 ret_val['doc_mag_std'] = np.std(mags)
 ret_val['sentiment_entities'] = []
ret_val['doc_ent_person_count'] = 0
ret_val['doc_ent_location_count'] = 0
ret_val['doc_ent_org_count'] = 0
ret_val['doc_ent_event_count'] = 0
ret_val['doc_ent_woa_count'] = 0
ret_val['doc_ent_good_count'] = 0
ret_val['doc_ent_other_count'] = 0
key_mapper = {
'PERSON':'doc_ent_person_count',
'LOCATION':'doc_ent_location_count',
'ORGANIZATION':'doc_ent_org_count',
'EVENT':'doc_ent_event_count',
'WORK_OF_ART':'doc_ent_woa_count',
'CONSUMER_GOOD':'doc_ent_good_count',
'OTHER':'doc_ent_other_count'
}
for e in file['entities']:
ret_val['sentiment_entities'].append(e['name'])
 if e['type'] in key_mapper:
ret_val[key_mapper[e['type']]]+=1
ret_val['sentiment_entities'] = ' '.join(ret_val['sentiment_entities'])
 return ret_val
def parse_metadata_file(self, file, img):

 file_keys = list(file.keys())
 if 'textAnnotations' in file_keys:
textblock_num = len(file['textAnnotations'])
 textlen = np.sum([len(text['description'])for text in file['textAnnotations']])
 else:
textblock_num = 0
textlen = 0
if 'faceAnnotations' in file_keys:
faceanno = 1
else:
faceanno = 0
if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
if len(file_annots)==0:
file_label_score_mean = np.nan
file_label_score_max = np.nan
file_label_score_min = np.nan
else:
temp = np.asarray([x['score'] for x in file_annots])
 file_label_score_mean = temp.mean()
file_label_score_max = temp.max()
file_label_score_min = temp.min()
file_top_desc = [x['description'] for x in file_annots]
else:
file_label_score_mean = np.nan
file_label_score_max = np.nan
file_label_score_min = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
if len(file_colors)==0:
file_color_score = np.nan
file_color_pixelfrac = np.nan
color_red_mean = np.nan
color_green_mean = np.nan
color_blue_mean = np.nan
color_red_std = np.nan
color_green_std = np.nan
color_blue_std = np.nan
else:
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_red = np.asarray([x['color']['red'] if 'red' in x['color'] else 0 for x in file_colors])
 file_color_green = np.asarray([x['color']['green'] if 'green' in x['color'] else 0for x in file_colors])
 file_color_blue = np.asarray([x['color']['blue'] if 'blue' in x['color'] else 0 for x in file_colors])
 color_red_mean = file_color_red.mean()
color_green_mean = file_color_green.mean()
color_blue_mean = file_color_blue.mean()
color_red_std = file_color_red.std()
color_green_std = file_color_green.std()
color_blue_std = file_color_blue.std()
if len(file_crops)==0:
file_crop_conf=np.nan
file_crop_importance = np.nan
file_crop_fraction_mean = np.nan
file_crop_fraction_sum = np.nan
file_crop_fraction_std = np.nan
file_crop_num = 0
else:
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
file_crop_num = len(file_crops)
 if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
crop_areas = []
image_area = img.shape[0]*img.shape[1]
for crophint in file_crops:
v_x, v_y = [], []
for vertices in crophint['boundingPoly']['vertices']:
if 'x' not in vertices:
v_x.append(0)
 else:
v_x.append(vertices['x'])
 if 'y' not in vertices:
v_y.append(0)
 else:
v_y.append(vertices['y'])
 crop_areas.append(( max(v_x)-min(v_x)) *(max(v_y)-min(v_y)) /image_area)
 file_crop_fraction_mean = np.mean(crop_areas)
 file_crop_fraction_sum = np.sum(crop_areas)
 file_crop_fraction_std = np.std(crop_areas)
 df_metadata = {
'label_score_mean': file_label_score_mean,
'label_score_max': file_label_score_max,
'label_score_min': file_label_score_min,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'color_red_mean':color_red_mean,
'color_green_mean':color_green_mean,
'color_blue_mean':color_blue_mean,
'color_red_std':color_red_std,
'color_green_std':color_green_std,
'color_blue_std':color_blue_std,
'crop_area_sum':file_crop_fraction_sum,
'annots_top_desc': self.sentence_sep.join(file_top_desc),
'img_aratio':img.shape[0]/img.shape[1],
'text_len':textlen,
'textblock_num':textblock_num,
'face_annotation':faceanno
}
return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = None
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 image_filename = '.. /input/petfinder-adoption-prediction/{}_images/{}-{}.jpg'.format(mode, pet_id, ind)
 try:
image = cv2.imread(image_filename)
 metadata_file = pet_parser.open_metadata_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file, image)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
return [df_sentiment, dfs_metadata]
pet_parser = PetFinderParser()<create_dataframe>
def cleanTicket(ticket):
ticket = ticket.replace('.', '')
 ticket = ticket.replace('/', '')
 ticket = ticket.split()
ticket = map(lambda t : t.strip() , ticket)
 ticket = list(filter(lambda t : not t.isdigit() , ticket))
if len(ticket)> 0:
return ticket[0]
else:
return 'XXX'
dfs_test = Parallel(n_jobs=n_jobs, verbose=1 )(
 delayed(extract_additional_features )(i, mode='test')for i in test_pet_ids)
 test_dicts_sentiment = [x[0] for x in dfs_test if x[0] is not None]
test_dfs_metadata = [x[1] for x in dfs_test if len(x[1])>0]
test_dfs_sentiment = pd.DataFrame(test_dicts_sentiment)
 test_dfs_metadata = list(itertools.chain.from_iterable(test_dfs_metadata))
test_dfs_metadata = pd.DataFrame(test_dfs_metadata)
 print(test_dfs_sentiment.shape, test_dfs_metadata.shape )<concatenate>
tickets = set()
for t in combined['Ticket']:
tickets.add(cleanTicket(t))
meta_df = pd.concat([train_dfs_metadata, test_dfs_metadata], sort=False ).reset_index(drop=True)
 senti_df = pd.concat([train_dfs_sentiment, test_dfs_sentiment], sort=False ).reset_index(drop=True )<feature_engineering>
def process_ticket() :
global combined
def cleanTicket(ticket):
ticket = ticket.replace('.','')
 ticket = ticket.replace('/','')
 ticket = ticket.split()
ticket = map(lambda t : t.strip() , ticket)
 ticket = filter(lambda t : not t.isdigit() , ticket)
 if len(ticket)> 0:
return ticket[0]
else:
return 'XXX'
metadata_desc = meta_df.groupby(['PetID'])['annots_top_desc'].unique().reset_index()
metadata_desc['meta_annots_top_desc'] = metadata_desc['annots_top_desc'].apply(lambda x: '; '.join(x))
metadata_desc.drop('annots_top_desc', axis=1, inplace=True)
 possible_annots = set()
for i in range(len(meta_df)) :
possible_annots = possible_annots.union(set(meta_df['annots_top_desc'].iloc[i].split('; ')))
 annot_mapper = {}
for idx, a in enumerate(possible_annots):
annot_mapper[a] = str(idx)
 metadata_desc['meta_desc'] = metadata_desc['meta_annots_top_desc'].apply(lambda x: ' '.join(annot_mapper[i] for i in x.split('; ')) )<feature_engineering>
combined['Ticket'] = combined['Ticket'].map(cleanTicket)
 tickets_dummies = pd.get_dummies(combined['Ticket'], prefix='Ticket')
 combined = pd.concat([combined, tickets_dummies], axis=1)
 combined.drop('Ticket', inplace=True, axis=1 )
senti_df['sentiment_entities'].fillna('', inplace=True)
 senti_df['sentiment_entities'] = senti_df['sentiment_entities'].str.lower()
senti_df['sentiment_len'] = senti_df['sentiment_entities'].apply(lambda x:len(x))
senti_df['sentiment_word_len'] = senti_df['sentiment_entities'].apply(lambda x: len(x.replace(';',' ' ).split(' ')))
 senti_df['sentiment_word_unique'] = senti_df['sentiment_entities'].apply(lambda x: len(set(x.replace(';',' ' ).split(' '))))
senti_df['doc_language'] = pd.factorize(senti_df['doc_language'])[0]<define_variables>
combined.drop('Name', inplace=True, axis=1)
 combined.drop('SibSp', inplace=True, axis=1)
 combined.drop('family_size', inplace=True, axis=1)
 combined.drop('Cabin', inplace=True, axis=1 )
aggregates = {
'color_blue_mean':['mean','std'],
'color_blue_std':['mean'],
'color_green_mean':['mean','std'],
'color_green_std':['mean'],
'color_pixelfrac':['mean','std'],
'color_red_mean':['mean','std'],
'color_red_std':['mean'],
'color_score':['mean','max'],
'crop_area_sum':['mean','std','min'],
'crop_conf':['mean','std','max'],
'crop_importance':['mean','std'],
'label_score_max':['mean','std','max'],
'label_score_mean':['mean','max','std'],
'label_score_min':['mean','max','std'],
'img_aratio':['nunique','std','max','min'],
'textblock_num':['mean','max'],
'face_annotation':['mean','nunique']
}
metadata_gr = meta_df.drop(['annots_top_desc'], axis=1)
 for i in metadata_gr.columns:
if 'PetID' not in i:
metadata_gr[i] = metadata_gr[i].astype(float)
 metadata_gr = metadata_gr.groupby(['PetID'] ).agg(aggregates)
 metadata_gr.columns = pd.Index(['{}_{}_{}'.format('meta', c[0], c[1].upper())for c in metadata_gr.columns.tolist() ])
 metadata_gr = metadata_gr.reset_index()<merge>
title_dummies = pd.get_dummies(combined['Title'], prefix='Title')
 combined = pd.concat([combined, title_dummies], axis=1)
 combined.drop('Title', inplace=True, axis=1 )
meta_df = metadata_desc.merge(metadata_gr, on='PetID', how='left' )<drop_column>
train1 = combined[:891]
test1 = combined[891:]
test1.drop(labels=["Survived"],axis = 1,inplace=True )
feat_df = data_df[['PetID','Color1','Breed1','State','RescuerID','Name','Breed_full','Color_full','hard_interaction']]<merge>
train1["Survived"] = train1["Survived"].astype(int)
 Y_train = train1["Survived"]
X_train = train1.drop(labels = ["Survived"],axis = 1 )
agg = {
'Fee':['mean','std','max'],
'avg_fee':['mean','std','max'],
'Breed1':['nunique'],
'Age':['mean','std','max'],
'Quantity':['std'],
'PetID':['nunique']
}
feat = data_df.groupby('Color1' ).agg(agg)
 feat.columns = pd.Index(['COLOR_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Color1', how='left')
 agg = {
'Fee':['mean','std','max'],
'avg_fee':['mean','std','max'],
'Breed_full':['nunique'],
'Quantity':['sum'],
}
feat = data_df.groupby('Color_full' ).agg(agg)
 feat.columns = pd.Index(['COLORfull_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Color_full', how='left' )<groupby>
train1 = combined[:891]
test1 = combined[891:]
test1.drop(labels=["Survived"],axis = 1,inplace=True )
agg = {
'Color_full':['nunique'],
'Breed2':['nunique'],
'FurLength':['nunique'],
'Fee':['mean','max'],
'avg_fee':['mean','std','max'],
'Age':['mean','std','min','max'],
'Quantity':['mean','std','max','sum'],
'PetID':['nunique'],
'FurLength':['mean'],
'Health':['mean'],
'MaturitySize':['mean','std','min','max'],
'Vaccinated':['mean'],
'Dewormed':['mean'],
'Sterilized':['mean']
}
feat = data_df.groupby('Breed1' ).agg(agg)
 feat.columns = pd.Index(['BREED1_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Breed1', how='left')
 agg = {
'Color_full':['nunique'],
'Fee':['mean','min','max'],
'avg_fee':['mean','std','max'],
'Quantity':['sum'],
'PetID':['nunique']
}
feat = data_df.groupby('Breed_full' ).agg(agg)
 feat.columns = pd.Index(['BREEDfull_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='Breed_full', how='left' )<define_variables>
train1["Survived"] = train1["Survived"].astype(int)
 Y_train = train1["Survived"]
X_train = train1.drop(labels = ["Survived"],axis = 1 )
agg = {
'Color_full':['nunique'],
'Breed_full':['nunique'],
'PetID':['nunique'],
'RescuerID':['nunique'],
'Fee':['mean','max'],
'avg_fee':['mean','std','max'],
'Age':['mean','std','max'],
'Quantity':['mean','std','max'],
'FurLength':['mean','std'],
'Health':['mean'],
'MaturitySize':['mean','std'],
'Vaccinated':['mean'],
'Dewormed':['mean'],
'Sterilized':['mean'],
'VideoAmt':['mean','std'],
'PhotoAmt':['mean','std'],
'avg_photo':['mean','std']
}
feat = data_df.groupby('State' ).agg(agg)
 feat.columns = pd.Index(['STATE_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='State', how='left' )<merge>
clf = RandomForestClassifier(n_estimators=25, max_features='sqrt')
 clf = clf.fit(X_train, Y_train)
 features = pd.DataFrame()
features['feature'] = X_train.columns
features['importance'] = clf.feature_importances_
features.sort_values(by=['importance'], ascending=True, inplace=True)
 features.set_index('feature', inplace=True )
feat = data_df.groupby('Name')['PetID'].agg({'name_count':'nunique'} ).reset_index()
feat_df = feat_df.merge(feat, on='Name', how='left' )<groupby>
model = SelectFromModel(clf, prefit=True)
 X_train_reduced = model.transform(X_train)
 print(X_train_reduced.shape )
agg = {
'PetID':['nunique'],
'Breed_full':['nunique'],
'VideoAmt':['mean','std'],
'PhotoAmt':['mean','std'],
'avg_photo':['mean','std'],
'Sterilized':['mean'],
'Dewormed':['mean'],
'Vaccinated':['mean']
}
rescuer_count = data_df.groupby(['RescuerID'] ).agg(agg)
 rescuer_count.columns = pd.Index(['RESCUER_' + e[0] + "_" + e[1].upper() for e in rescuer_count.columns.tolist() ])
 rescuer_count.reset_index(inplace=True)
 feat_df = feat_df.merge(rescuer_count, how='left', on='RescuerID' )<merge>
logreg = LogisticRegression()
logreg_cv = LogisticRegressionCV()
rf = RandomForestClassifier()
gboost = GradientBoostingClassifier()
models = [logreg, logreg_cv, rf, gboost]
agg = {
'Fee':['mean','min','max'],
'avg_fee':['mean','std','max']
}
feat = data_df.groupby('hard_interaction' ).agg(agg)
 feat.columns = pd.Index(['INTERACTION_' + e[0] + "_" + e[1].upper() for e in feat.columns.tolist() ])
 feat.reset_index(inplace=True)
 feat_df = feat_df.merge(feat, on='hard_interaction', how='left' )<drop_column>
def compute_score(clf, X, y, scoring='accuracy'):
xval = cross_val_score(clf, X, y, cv = 5, scoring=scoring)
 return np.mean(xval )
feat_df.drop(['Color1','Breed1','State','RescuerID','Name','Breed_full','Color_full','hard_interaction'], axis=1, inplace=True )<merge>
for model in models:
print('Cross-validation of : {0}'.format(model.__class__))
score = compute_score(clf=model, X=X_train, y=Y_train, scoring='accuracy')
 print('CV score = {0}'.format(score))
print('****' )
X_text = data_df[['PetID','Description']].merge(data_df[['PetID','Chinese_desc']], on='PetID', how='left')
 X_text = X_text.merge(senti_df[['PetID','sentiment_entities']], on='PetID', how='left')
 X_text = X_text.merge(metadata_desc[['PetID','meta_annots_top_desc','meta_desc']], on='PetID', how='left')
 text_columns = ['Description','Chinese_desc','sentiment_entities','meta_annots_top_desc','meta_desc']
print(text_columns)
 X_text['meta_annots_top_desc'].fillna(' ',inplace=True)
 X_text['meta_desc'].fillna(' ',inplace=True)
 X_text['sentiment_entities'].fillna(' ',inplace=True )<define_search_space>
parameters = {'bootstrap': False, 'min_samples_leaf': 4, 'n_estimators': 50,
'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 5}
model = RandomForestClassifier(**parameters)
 model.fit(X_train_reduced, Y_train )
text_features = []
ngram_ranges = [(1,3),(1,3),(1,1),(1,3),(1,1)]
n_components = [80, 24, 10, 32,16]
for idx, i in enumerate(text_columns):
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components[idx], random_state=1337)
 tfidf_col = TfidfVectorizer(ngram_range = ngram_ranges[idx], stop_words = 'english',
tokenizer=custom_tokenizer,
strip_accents='unicode' ).fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
text_features.append(svd_col)
<create_dataframe>
output = model.predict(test1_reduced ).astype(int)
 model1 = round(model.score(X_train_reduced, Y_train)* 100, 2)
 model1
docs = [custom_tokenizer(x)for x in data_df['Description'].values]
dictionary = corpora.Dictionary(docs)
 name = 'Description'
corpus = [dictionary.doc2bow(text)for text in docs]
lda = LdaMulticore(corpus, id2word=dictionary, num_topics=20, random_state = 999)
 docres = [dict(lda[doc_bow])for doc_bow in corpus]
doc_df = pd.DataFrame(docres,dtype=np.float16 ).fillna(0.001)
 doc_df.columns = ['%s_lda_%d'%(name,x)for x in range(20)]<concatenate>
output = model.predict(test1_reduced ).astype(int)
 submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived":output
})
 submission.to_csv("titanic51_submission.csv", index=False )