Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<categorify>
<drop_column>
warnings.filterwarnings('ignore')
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
with torch.no_grad() :
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
with torch.no_grad() :
for data in dataloader:
inputs = data['x'].to(device)
 outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
seed_everything(seed=42 )
feat_drop = ['Dewormed', 'Vaccinated', 'Sterilized']
train.drop(feat_drop, axis=1, inplace=True)
 train.shape<feature_engineering>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
<load_from_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv' )
test = pd.read_csv('.. /input/test/test.csv')
 test.drop(to_drop, axis=1, inplace=True)
 test.head()<categorify>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
<feature_engineering>
class DenoisingAutoEncoder(nn.Module):
def __init__(self, in_out_channels):
super(DenoisingAutoEncoder, self ).__init__()
self.encoder=nn.Sequential(
 nn.Linear(in_out_channels,256),
nn.ReLU(True),
nn.Linear(256,128),
nn.ReLU(True),
nn.Linear(128,64),
nn.ReLU(True)
)
 self.decoder=nn.Sequential(
 nn.Linear(64,128),
nn.ReLU(True),
nn.Linear(128,256),
nn.ReLU(True),
nn.Linear(256,in_out_channels), 
)
 def forward(self,x):
x=self.encoder(x)
 x=self.decoder(x)
 return x
class DAEDataset:
def __init__(self, ids, ys=None):
self.ids = ids
self.ys = ys
def __len__(self):
return len(self.ids)
 def __getitem__(self, item):
if self.ys is not None:
return {
'ids': torch.tensor(self.ids[item], dtype=torch.float32),
'targets': torch.tensor(self.ys[item], dtype=torch.float32),
}
else:
return {
'ids': torch.tensor(self.ids[item], dtype=torch.float32),
}
def dae_test_fn(data_loader, model, device):
model.eval()
preds = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"].to(device, dtype=torch.float32)
 outputs = model.encoder(ids)
 outputs = outputs.cpu().detach().numpy()
preds.append(outputs)
 preds = np.concatenate(preds, 0)
 return preds
<load_from_csv>
data_train = train_features[GENES+CELLS]
data_test = test_features[GENES+CELLS]
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES))
model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v3/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 model = DenoisingAutoEncoder(len(CELLS))
model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v3/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device )
<load_from_csv>
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape
<import_modules>
n_comp_GENES = 450
n_comp_CELLS = 2
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 train_features = pd.concat(( train_features, ae_genes_train, ae_cells_train), axis=1)
 test_features = pd.concat(( test_features, ae_genes_test, ae_cells_test), axis=1)
 train_features.shape, test_features.shape
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import make_scorer
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier<compute_train_metric>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
def Cmatrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = Cmatrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<split>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = pd.merge(train, train_drug, on='sig_id')
 groups = np.array(train.drug_id.values)
 folds = train.copy()
n_folds = 7
mskf = MultilabelStratifiedKFold(n_splits=n_folds)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target, groups=groups)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
len(feature_cols )
X, y, nan = proc_df(train, 'AdoptionSpeed')
<choose_model_class>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = n_folds
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024
skf = StratifiedKFold(n_splits=5, shuffle=True )<define_search_space>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, div_factor=1.5e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
max_depth_values = range(3, 9)
 min_child_weight = range(1, 3)
 n_estimators = [80]
xgb_params = {
'max_depth' : max_depth_values,
'min_child_weight' : min_child_weight,
'n_estimators' : n_estimators,
}<train_on_grid>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
%%time
xgb = XGBClassifier()
xgb_grid = GridSearchCV(xgb, xgb_params, n_jobs=-1, scoring=kappa_scorer, cv=skf)
 xgb_grid.fit(X, y);<find_best_params>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.25)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
xgb_grid.best_params_, xgb_grid.best_score_<predict_on_test>
SEED = [6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED )
xgb_grid.best_estimator_.predict(test )<save_to_csv>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof6.csv', index=False)
 test[target_cols] = predictions
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
sub = pd.read_csv('.. /input/test/sample_submission.csv')
 pred = xgb_grid.predict(test)
 sub['AdoptionSpeed'] = pd.Series(pred)
 sub.to_csv('submission.csv', index=False )<import_modules>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission6.csv', index=False )
import numpy as np
import pandas as pd
import json
from pandas.io.json import json_normalize
import os
from sklearn.model_selection import StratifiedKFold
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import cohen_kappa_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt<load_from_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 print(train_drug.shape )
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
<load_from_disk>
def rank_gauss(data):
epsilon = 1e-6
for k in tqdm(GENES + CELLS):
r_cpu = data.loc[:,k].argsort().argsort()
r_cpu =(r_cpu/r_cpu.max() -0.5)*2
r_cpu = np.clip(r_cpu,-1+epsilon,1-epsilon)
 r_cpu = sp_erfinv(r_cpu)
 data.loc[:,k] = r_cpu * np.sqrt(2)
 return data
class DenoisingAutoEncoder(nn.Module):
def __init__(self, in_out_channels, hidden_channels):
super(DenoisingAutoEncoder, self ).__init__()
self.encoder=nn.Sequential(
 nn.Linear(in_out_channels, hidden_channels),
nn.ReLU(True),
nn.Linear(hidden_channels, hidden_channels//2),
nn.ReLU(True),
nn.Linear(hidden_channels//2, hidden_channels//4),
nn.ReLU(True)
)
 self.decoder=nn.Sequential(
 nn.Linear(hidden_channels//4, hidden_channels//2),
nn.ReLU(True),
nn.Linear(hidden_channels//2, hidden_channels),
nn.ReLU(True),
nn.Linear(hidden_channels, in_out_channels), 
)
 def forward(self,x):
x=self.encoder(x)
 x=self.decoder(x)
 return x
def readFile(fn):
file = '.. /input/train_sentiment/'+fn['PetID']+'.json'
if os.path.exists(file):
with open(file)as data_file:
data = json.load(data_file)
 df = json_normalize(data)
 mag = df['documentSentiment.magnitude'].values[0]
score = df['documentSentiment.score'].values[0]
return pd.Series([mag,score],index=['mag','score'])
 else:
return pd.Series([0,0],index=['mag','score'])
 def readTestFile(fn):
file = '.. /input/test_sentiment/'+fn['PetID']+'.json'
if os.path.exists(file):
with open(file)as data_file:
data = json.load(data_file)
 df = json_normalize(data)
 mag = df['documentSentiment.magnitude'].values[0]
score = df['documentSentiment.score'].values[0]
return pd.Series([mag,score],index=['mag','score'])
 else:
return pd.Series([0,0],index=['mag','score'] )<feature_engineering>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_features = rank_gauss(train_features)
 test_features = rank_gauss(test_features)
 data_train = train_features[GENES+CELLS]
data_test = test_features[GENES+CELLS]
train[['SentMagnitude', 'SentScore']] = train[['PetID']].apply(lambda x: readFile(x), axis=1)
 test[['SentMagnitude', 'SentScore']] = test[['PetID']].apply(lambda x: readTestFile(x), axis=1)
<prepare_x_and_y>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES), 512)
 model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v5/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 print(ae_genes_train_reconstruction.shape)
 print(ae_genes_test_reconstruction.shape )
train_X = train.drop(['Name', 'Description', 'RescuerID', 'PetID', 'AdoptionSpeed'], axis=1)
 train_y = train['AdoptionSpeed']
test_X = test.drop(['Name', 'Description', 'RescuerID', 'PetID'], axis=1)
<train_model>
model = DenoisingAutoEncoder(len(CELLS), 64)
 model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v5/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device)
 print(ae_cells_train_reconstruction.shape)
 print(ae_cells_test_reconstruction.shape )
train_meta = np.zeros(train_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 clf = RandomForestClassifier(bootstrap=True, criterion = 'gini', max_depth=80,
max_features='auto', min_samples_leaf=5,
min_samples_split=5, n_estimators=200)
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=1812 ).split(train_X, train_y))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = train_X.iloc[train_idx]
y_train = train_y[train_idx]
X_val = train_X.iloc[valid_idx]
y_val = train_y[valid_idx]
clf.fit(X_train, y_train)
 y_pred = clf.predict(X_val)
 print("Fold {}: accuracy = {:.1f}%, kappa = {:.4f}(no boundary adjustment)".format(idx,
100.0*accuracy_score(y_val, y_pred),
cohen_kappa_score(y_val, y_pred, weights='quadratic')))
 y_probs = clf.predict_proba(X_val)
 class_vals = clf.classes_.copy()
class_vals[0] = -1
y_floats = [sum(y_probs[ix]*class_vals)for ix in range(len(y_probs[:,0])) ]
train_meta[valid_idx] = y_floats
test_probs = clf.predict_proba(test_X)
 test_meta += np.array([sum(test_probs[ix]*class_vals)for
ix in range(len(test_probs[:,0])) ])/ len(splits )<predict_on_test>
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64//4)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(512//4)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64//4)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(512//4)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape
def get_class_bounds(y, y_pred, N=5, class0_fraction=-1):

 ysort = np.sort(y)
 predsort = np.sort(y_pred)
 bounds = []
for ibound in range(N-1):
iy = len(ysort[ysort <= ibound])
 if(ibound == 0)and(class0_fraction >= 0.0):
iy = int(class0_fraction * iy)
 bounds.append(predsort[iy])
 return bounds
def assign_class(y_pred, boundaries):

 y_classes = np.zeros(len(y_pred))
for iclass, bound in enumerate(boundaries):
y_classes[y_pred >= bound] = iclass + 1
return y_classes.astype(int )<statistical_test>
n_comp_GENES = 450
n_comp_CELLS = 2
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
kappas = []
mses = []
accurs = []
cl0fracs = np.array(np.arange(0.01,1.001,0.01))
for cl0frac in cl0fracs:
boundaries = get_class_bounds(train_y, train_meta, class0_fraction=cl0frac)
 train_meta_ints = assign_class(train_meta, boundaries)
 kappa = cohen_kappa_score(train['AdoptionSpeed'], train_meta_ints, weights='quadratic')
 kappas.append(kappa)
 mse = mean_squared_error(train['AdoptionSpeed'], train_meta_ints)
 mses.append(mse)
 accur = accuracy_score(train['AdoptionSpeed'], train_meta_ints)
 accurs.append(accur)
 ifmax = np.array(kappas ).argmax()
cl0frac = cl0fracs[ifmax]
print("Best kappa for class0 fraction = {:.4f}".format(cl0frac))<compute_train_metric>
train_features = pd.concat(( train_features, ae_genes_train, ae_cells_train), axis=1)
 test_features = pd.concat(( test_features, ae_genes_test, ae_cells_test), axis=1)
 train_features.shape, test_features.shape
print("Using class0_fraction = {:.4f}, gives boundaries:".format(cl0frac))
boundaries = get_class_bounds(train_y, train_meta, class0_fraction=cl0frac)
 print(boundaries)
 train_meta_ints = assign_class(train_meta, boundaries)
 kappa = cohen_kappa_score(train_y, train_meta_ints, weights='quadratic')
 print("Adjusted boundaries give:")
 print("kappa = {:.4f}(with accuracy = {:.1f}%)".format(kappa,
100.0*accuracy_score(train_y, train_meta_ints)) )<compute_test_metric>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
con_mat = confusion_matrix(train_y, train_meta_ints)
 diag = 0.0
for id in range(5):
diag += con_mat[id,id]
print("
Confusion matrix - Columns are prediced 0, predicted 1, etc.
 ")
 print(con_mat)
 print("")
 print("
{2:.2f}% = {0}/{1} are on the diagonal(= accuracy)".format(
 int(diag), con_mat.sum() , 100.0*diag/con_mat.sum()))<save_to_csv>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
sub = pd.read_csv('.. /input/test/sample_submission.csv')
 sub['AdoptionSpeed'] = test_meta_ints
sub['AdoptionSpeed'] = sub['AdoptionSpeed'].astype(int)
 sub.to_csv("submission.csv", index=False)
<load_from_csv>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 train_df.info()<load_from_csv>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 test_df.info()<load_from_csv>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
breed_label = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 breed_label.head()<feature_engineering>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
train_df['Mixed_Breed'] = train_df.apply(lambda x: 0 if x.Breed2==0 and x.Breed1!=307 else 1, axis=1)
 test_df['Mixed_Breed'] = test_df.apply(lambda x: 0 if x.Breed2==0 and x.Breed1!=307 else 1, axis=1 )<load_from_csv>
model = Model(num_features, num_all_targets)
 model
color_label = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 color_label<feature_engineering>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
train_df['Num_Color'] = train_df.apply(lambda x: 3-sum([y==0 for y in [x.Color1, x.Color2, x.Color3]]), axis=1)
 test_df['Num_Color'] = test_df.apply(lambda x: 3-sum([y==0 for y in [x.Color1, x.Color2, x.Color3]]), axis=1 )<load_from_csv>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
state_label = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 state_label<categorify>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
train_df['Description_Length'] = train_df.Description.map(len)
 test_df['Description_Length'] = test_df.Description.map(len )<define_variables>
SEED = [0, 2, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
sentiment_list = os.listdir('.. /input/petfinder-adoption-prediction/train_sentiment' )<load_from_disk>
str(timedelta(seconds=time_diff))
example_text = pd.read_json('.. /input/petfinder-adoption-prediction/train_sentiment/{}'.format(sentiment_list[1000]), orient='index', typ='series' )<feature_engineering>
train[['sig_id']+target_cols].to_csv('oof7.csv', index=False)
 valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
def get_desc_stats(desc_json):
example = pd.read_json('.. /input/petfinder-adoption-prediction/train_sentiment/{}'.format(desc_json), orient='index', typ='series')
 result = {}
result['num_sentences'] = len(example.sentences)
 result['num_entities'] = len(example.entities)
 result['magnitude'] = example.documentSentiment['magnitude']
result['score'] = example.documentSentiment['score']
return result<feature_engineering>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission7.csv', index=False )
sentiment = {}
for x in sentiment_list:
sentiment[x[:9]] = get_desc_stats(x )<create_dataframe>
class DenoisingAutoEncoder(nn.Module):
def __init__(self, in_out_channels):
super(DenoisingAutoEncoder, self ).__init__()
self.encoder=nn.Sequential(
 nn.Linear(in_out_channels,256),
nn.ReLU(True),
nn.Linear(256,128),
nn.ReLU(True),
nn.Linear(128,64),
nn.ReLU(True)
)
 self.decoder=nn.Sequential(
 nn.Linear(64,128),
nn.ReLU(True),
nn.Linear(128,256),
nn.ReLU(True),
nn.Linear(256,in_out_channels)
)
 def forward(self,x):
x=self.encoder(x)
 x=self.decoder(x)
 return x
sentiment_df = pd.DataFrame.from_dict(sentiment ).transpose()<merge>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 train_features = pd.merge(train_features, train_drug, on='sig_id')
 train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
groups = np.array(train_features.drug_id.values)
 del train_features['drug_id']
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
train_df = train_df.join(sentiment_df, on='PetID' )<define_search_space>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
scaler = preprocessing.StandardScaler()
scaler.fit(train_features[GENES+CELLS].append(test_features[GENES+CELLS]))
data_train = pd.DataFrame(scaler.transform(train_features[GENES+CELLS]), columns=GENES+CELLS)
 data_test = pd.DataFrame(scaler.transform(test_features[GENES+CELLS]), columns=GENES+CELLS)
 data_train.shape, data_test.shape
train_df.magnitude.fillna(-1, inplace=True)
 train_df.score.fillna(-1, inplace=True)
 train_df.num_sentences.fillna(-1, inplace=True)
 train_df.num_entities.fillna(-1, inplace=True )<feature_engineering>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES))
model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 print(ae_genes_train_reconstruction.shape)
 print(ae_genes_test_reconstruction.shape)
 model = DenoisingAutoEncoder(len(CELLS))
model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device)
 print(ae_cells_train_reconstruction.shape)
 print(ae_cells_test_reconstruction.shape )
sentiment_list_test = os.listdir('.. /input/petfinder-adoption-prediction/test_sentiment')
 def get_desc_stats_test(desc_json):
example = pd.read_json('.. /input/petfinder-adoption-prediction/test_sentiment/{}'.format(desc_json), orient='index', typ='series')
 result = {}
result['num_sentences'] = len(example.sentences)
 result['num_entities'] = len(example.entities)
 result['magnitude'] = example.documentSentiment['magnitude']
result['score'] = example.documentSentiment['score']
return result
sentiment_test = {}
for x in sentiment_list_test:
sentiment_test[x[:9]] = get_desc_stats_test(x)
 sentiment_df_test = pd.DataFrame.from_dict(sentiment_test ).transpose()
test_df = test_df.join(sentiment_df_test, on='PetID')
 test_df.magnitude.fillna(-1, inplace=True)
 test_df.score.fillna(-1, inplace=True)
 test_df.num_sentences.fillna(-1, inplace=True)
 test_df.num_entities.fillna(-1, inplace=True )<import_modules>
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape
from sklearn.feature_extraction.text import TfidfVectorizer<define_variables>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pattern = re.compile('[\W_]+', re.UNICODE)
 texts = [pattern.sub(' ', x)for x in train_df.Description]
texts_test = [pattern.sub(' ', x)for x in test_df.Description]<choose_model_class>
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train, test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
Tfidf = TfidfVectorizer(stop_words='english',ngram_range=(1,3), max_features=20000,max_df=0.95,min_df=5 )<categorify>
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
train_x_tfidf_full = Tfidf.fit_transform(texts)
 test_x_tfidf = Tfidf.transform(texts_test )<import_modules>
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
from sklearn.decomposition import TruncatedSVD<feature_engineering>
n_seeds = 3
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 7
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 oof_pred = np.zeros(( n_train,n_labels))
hists = []
for seed in seeds:
fold = 0
kf = MultilabelStratifiedKFold(n_splits=n_folds)
 for train, test in kf.split(X=train_features, y=labels_train, groups=groups):
X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
X_train_2 = np.concatenate([X_train_2,
ae_genes_train.iloc[train, :].values,
ae_cells_train.iloc[train, :].values], 1)
 X_test_2 = np.concatenate([X_test_2,
ae_genes_train.iloc[test, :].values,
ae_cells_train.iloc[test, :].values], 1)
 data_test_2 = np.concatenate([data_test_2,
ae_genes_test.values,
ae_cells_test.values], 1)
 y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=192,verbose=1,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('TwoHeads_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 oof_pred[test] += y_val/(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
svd = TruncatedSVD(n_components=100, random_state=42)
 train_x_svd = svd.fit_transform(train_x_tfidf_full )<merge>
oof_train = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv' ).loc[non_ctl_idx, :].reset_index(drop=True)
 oof_train.iloc[:, 1:] = oof_pred
oof_train.to_csv('oof8.csv', index=False )
train_df = train_df.join(pd.DataFrame(train_x_svd, columns=['svd_'+str(x)for x in np.arange(100)]))<merge>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission8.csv', index=False )
test_x_svd = svd.transform(test_x_tfidf)
 test_df = test_df.join(pd.DataFrame(test_x_svd, columns=['svd_'+str(x)for x in np.arange(100)]))<import_modules>
from sklearn.feature_extraction.text import CountVectorizer<feature_engineering>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_features2=train_features.copy()
test_features2=test_features.copy()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
cv = CountVectorizer(stop_words='english',ngram_range=(1,3), max_features=20000,max_df=0.95,min_df=5 )<categorify>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
train_x_cv = cv.fit_transform(texts)
 test_x_cv = cv.transform(texts_test )<create_dataframe>
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )
def wm2df(wm, feat_names):
df = pd.DataFrame(data=wm.toarray() ,
columns=feat_names)
 return(df )<categorify>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
svd_cv = TruncatedSVD(n_components=100, random_state=42)
 train_x_svd_cv = svd_cv.fit_transform(train_x_cv)
 train_df = train_df.join(pd.DataFrame(train_x_svd_cv, columns=['svd_cv_'+str(x)for x in np.arange(100)]))
test_x_svd_cv = svd_cv.transform(test_x_cv)
 test_df = test_df.join(pd.DataFrame(test_x_svd_cv, columns=['svd_cv_'+str(x)for x in np.arange(100)]))<categorify>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
def isChinese(s):
if len(re.findall(u'[\u4e00-\u9fff]', s)) > 0:
return 1
else:
return 0<categorify>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
train_df['contains_chinese'] = train_df.Description.map(isChinese)
 test_df['contains_chinese'] = test_df.Description.map(isChinese )<count_values>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
train_df.contains_chinese.value_counts()<define_variables>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
photo_list = os.listdir('.. /input/petfinder-adoption-prediction/train_metadata' )<define_variables>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
profile_photo = [x for x in photo_list if "-1." in x]<feature_engineering>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1)
 train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'])
 feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
def get_dominant_color(photo_json):
example = pd.read_json('.. /input/petfinder-adoption-prediction/train_metadata/{}'.format(photo_json), orient='index', typ='series')
 max_index = np.argmax([x['pixelFraction'] for x in example.imagePropertiesAnnotation['dominantColors']['colors']])
 result = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['color']
result['score'] = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['score']
result['pixelFraction'] = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['pixelFraction']
try:
result['image_description'] = example.labelAnnotations[0]['description']
result['image_description_score'] = example.labelAnnotations[0]['score']
except AttributeError:
result['image_description'] = -1
result['image_description_score'] = -1
result['image_confidence'] = example.cropHintsAnnotation['cropHints'][0]['confidence']
try:
result['image_importanceFraction'] = example.cropHintsAnnotation['cropHints'][0]['importanceFraction']
except KeyError:
result['image_importanceFraction'] = -1
return result<feature_engineering>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
dominant_color_train = {}
for x in profile_photo:
dominant_color_train[x[:9]] = get_dominant_color(x )<merge>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
dominant_color_df = pd.DataFrame(dominant_color_train ).transpose()
dominant_color_df.columns = ['photo_'+x for x in dominant_color_df.columns]
train_df = train_df.join(dominant_color_df, on='PetID', rsuffix='_color' )<data_type_conversions>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
train_df[pd.DataFrame(dominant_color_df ).columns.tolist() ] = train_df[pd.DataFrame(dominant_color_df ).columns.tolist() ].fillna(-1 )<feature_engineering>
SEED = [77, 777]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions
photo_list_test = os.listdir('.. /input/petfinder-adoption-prediction/test_metadata')
 profile_photo_test = [x for x in photo_list_test if "-1." in x]
def get_dominant_color_test(photo_json):
example = pd.read_json('.. /input/petfinder-adoption-prediction/test_metadata/{}'.format(photo_json), orient='index', typ='series')
 max_index = np.argmax([x['pixelFraction'] for x in example.imagePropertiesAnnotation['dominantColors']['colors']])
 result = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['color']
result['score'] = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['score']
result['pixelFraction'] = example.imagePropertiesAnnotation['dominantColors']['colors'][max_index]['pixelFraction']
try:
result['image_description'] = example.labelAnnotations[0]['description']
result['image_description_score'] = example.labelAnnotations[0]['score']
except AttributeError:
result['image_description'] = -1
result['image_description_score'] = -1
result['image_confidence'] = example.cropHintsAnnotation['cropHints'][0]['confidence']
try:
result['image_importanceFraction'] = example.cropHintsAnnotation['cropHints'][0]['importanceFraction']
except KeyError:
result['image_importanceFraction'] = -1
return result<feature_engineering>
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]
dominant_color_test = {}
for x in profile_photo_test:
dominant_color_test[x[:9]] = get_dominant_color_test(x )<merge>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
dominant_color_df_test = pd.DataFrame(dominant_color_test ).transpose()
dominant_color_df_test.columns = ['photo_'+x for x in dominant_color_df_test.columns]
test_df = test_df.join(dominant_color_df_test, on='PetID', rsuffix='_color')
 test_df[pd.DataFrame(dominant_color_df_test ).columns.tolist() ] = test_df[pd.DataFrame(dominant_color_df_test ).columns.tolist() ].fillna(-1 )<feature_engineering>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
photo_image_description = {}
for i,x in enumerate(pd.concat([train_df.photo_image_description, test_df.photo_image_description] ).value_counts().index):
photo_image_description[x] = i
train_df.photo_image_description = train_df.photo_image_description.map(lambda x: photo_image_description[x])
 test_df.photo_image_description = test_df.photo_image_description.map(lambda x: photo_image_description[x] )<define_variables>
SEED = [77, 777]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
dog_cat = [photo_image_description[x] for x in ['dog',
'dog breed',
'dog like mammal',
'dog breed group',
'pug',
'bull terrier',
'siberian husky',
'beagle',
'street dog',
'boston terrier',
'pomeranian',
'harrier',
'czechoslovakian wolfdog',
'basset hound',
'volpino italiano']]<define_variables>
train[['sig_id']+target_cols].to_csv('oof9.csv', index=False)
 valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / len(target_cols)
 print("CV log_loss: ", score )
cat_cat = [photo_image_description[x] for x in ['cat',
'small to medium sized cats',
'cat like mammal',
'black cat']]<categorify>
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission9.csv',index=True )
def main_category(x):
if x in dog_cat:
return 1
elif x in cat_cat:
return 2
elif x == photo_image_description[-1]:
return 0
else:
return 3<categorify>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
train_df['photo_category'] = train_df.photo_image_description.map(main_category)
 test_df['photo_category'] = test_df.photo_image_description.map(main_category )<load_from_csv>
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
image_stat_train = pd.read_csv('.. /input/image-statistics-for-petfinder/train_image.csv')
 image_stat_train.info()<load_from_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_features2=train_features.copy()
test_features2=test_features.copy()
image_stat_test = pd.read_csv('.. /input/image-statistics-for-petfinder/test_image.csv')
 image_stat_test.info()<feature_engineering>
def rank_gauss(data):
epsilon = 1e-6
for k in tqdm(GENES + CELLS):
r_cpu = data.loc[:,k].argsort().argsort()
r_cpu =(r_cpu/r_cpu.max() -0.5)*2
r_cpu = np.clip(r_cpu,-1+epsilon,1-epsilon)
 r_cpu = sp_erfinv(r_cpu)
 data.loc[:,k] = r_cpu * np.sqrt(2)
 return data
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_features = rank_gauss(train_features)
 test_features = rank_gauss(test_features )
image_stat_train['PetID'] = image_stat_train.image.map(lambda x: x[:9])
 image_stat_test['PetID'] = image_stat_test.image.map(lambda x: x[:9] )<prepare_output>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
image_stat_train = image_stat_train.set_index('PetID')
 image_stat_test = image_stat_test.set_index('PetID' )<merge>
n_comp = 600
gpca= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/gpca.pkl', 'rb'))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
cpca= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/cpca.pkl', 'rb'))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 n_comp = 450
gsvd= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/gsvd.pkl', 'rb'))
train2=(gsvd.transform(train_features[GENES]))
test2 =(gsvd.transform(test_features[GENES]))
train_gsvd = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp)])
 test_gsvd = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gsvd), axis=1)
 test_features = pd.concat(( test_features, test_gsvd), axis=1)
 n_comp = 2
csvd= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/csvd.pkl', 'rb'))
train2=(csvd.transform(train_features[CELLS]))
test2 =(csvd.transform(test_features[CELLS]))
train_csvd = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp)])
 test_csvd = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_csvd), axis=1)
 test_features = pd.concat(( test_features, test_csvd), axis=1 )
train_df = train_df.join(image_stat_train, on='PetID', rsuffix='_image')
 test_df = test_df.join(image_stat_test, on='PetID', rsuffix='_image' )<import_modules>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
import lightgbm as lgb<define_variables>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_genes = load(open('.. /input/moa-tabnet-correct-rankgauss-svd/kmeans_genes.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
features = [x for x in train_df.columns if x not in ['Name', 'Type','RescuerID','AdoptionSpeed','Description','PetID','image','dullness_whiteness','temp_size','photo_image_description']]<split>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_cells = load(open('.. /input/moa-tabnet-correct-rankgauss-svd/kmeans_cells.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
df_train, df_val = train_test_split(train_df, test_size=0.3, random_state=420 )<import_modules>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
from sklearn.metrics import cohen_kappa_score<compute_test_metric>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans_pca = load(open('.. /input/moa-tabnet-correct-rankgauss-svd/kmeans_pca.pkl', 'rb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca)
 train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def kappa_scorer(pred, train_data):
length = len(train_data.get_label())
 pred_results = [[pred[x], pred[x+length*1], pred[x+length*2], pred[x+length*3], pred[x+length*4]] for x in np.arange(length)]
return 'kappa', cohen_kappa_score([np.argmax(x)for x in pred_results],train_data.get_label() , weights='quadratic'), True<create_dataframe>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
d_train = lgb.Dataset(df_train[features], label=df_train['AdoptionSpeed'],feature_name=features,
categorical_feature=['Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'State','Mixed_Breed','photo_category','contains_chinese'])
 d_val = lgb.Dataset(df_val[features], label=df_val['AdoptionSpeed'], reference=d_train,feature_name=features,
categorical_feature=['Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'State', 'Mixed_Breed','photo_category','contains_chinese'])
 params = {"objective" : "multiclass",
"num_class": 5,
"metric" : "None",
"learning_rate" : 0.1,
"feature_fraction_seed" : 420,
"feature_fraction" : 0.4,
"early_stopping_rounds": 200
}
evals_result = {}
model = lgb.train(params, d_train, num_boost_round=2000, valid_sets=[d_train, d_val], feval=kappa_scorer, evals_result=evals_result, verbose_eval=50 )<create_dataframe>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1)
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
target=target[target_cols]
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'])
 feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
train = train[feature_cols]
test = test_[feature_cols]
d_train = lgb.Dataset(df_train[features], label=df_train['AdoptionSpeed'],feature_name=features,
categorical_feature=['Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'State','Mixed_Breed','photo_category','contains_chinese'])
 cv_dict = lgb.cv(params, d_train, num_boost_round=1000, feval=kappa_scorer, verbose_eval=50 )<train_model>
X_test = test.values
dummy_model = DummyClassifier(random_state=1)
 dummy_model.fit(df_train[features], df_train['AdoptionSpeed'])
 cohen_kappa_score(dummy_model.predict(df_val[features]), df_val['AdoptionSpeed'] )<predict_on_test>
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 5e-5)+ y_true * np.log(logits + 5e-5)
 return np.mean(-aux)
 MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
pred_test = model.predict(test_df[features], num_iteration=model.best_iteration )<concatenate>
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 SEED = [19, 94, 6, 7, 1, 8]
for s in SEED:
tabnet_params['seed'] = s
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, target)) :
model = TabNetRegressor()
model.load_model(f".. /input/moa-tabnet-correct-rankgauss-svd/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds )
submission = pd.concat([test_df.PetID,pd.DataFrame(pred_test, columns=['A','B','C','D','E'])],axis=1 )<prepare_output>
all_feat = [col for col in df.columns if col not in ["sig_id"]]
test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission10.csv", index = None)
 submission.head()
submission['AdoptionSpeed'] = submission.apply(lambda x: np.argmax([x.A,x.B,x.C,x.D,x.E]), axis=1 )<save_to_csv>
oof6 = pd.read_csv("oof6.csv")
 oof7 = pd.read_csv("oof7.csv")
 oof8 = pd.read_csv("oof8.csv")
 oof9 = pd.read_csv("oof9.csv")
 oof10 = pd.read_csv('.. /input/moa-tabnet-correct-rankgauss-svd/oof.csv')
 print(oof6.shape)
 print(oof7.shape)
 print(oof8.shape)
 print(oof9.shape)
 print(oof10.shape )
submission.to_csv('submission.csv',index=False )<count_values>
sub6 = pd.read_csv("submission6.csv")
 sub7 = pd.read_csv("submission7.csv")
 sub8 = pd.read_csv("submission8.csv")
 sub9 = pd.read_csv("submission9.csv")
 sub10 = pd.read_csv("submission10.csv" )
submission.AdoptionSpeed.value_counts()<define_variables>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape
class Columns(Enum):
rescuer_id = ["RescuerID"]
ind_cont_columns = ["Age", "Fee", "VideoAmt", "PhotoAmt","Quantity",
"DescScore", "DescMagnitude", "DescLength", "SentMagnitude", "SentMagnitute_Mean",
"SentScore", "SentScore_Mean", "EntSalience", "EntSalience_Mean",
"NameLength","Pet_Maturity", "Color_Type","Fee_Per_Pet"]
ind_num_cat_columns = ["Type", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3",
"Vaccinated", "Dewormed", "Sterilized", "Health", "State", "MaturitySize",
"FurLength", "Pet_Breed", "Breed_Merge", "Pet_Purity", "Overall_Status"]
ind_text_columns = ["Name", "Description", "Entities"]
iden_columns = ["PetID"]
dep_columns = ["AdoptionSpeed"]
n_desc_svd_comp = 120
desc_svd_cols = ["desc_svd_"+str(i)for i in range(n_desc_svd_comp)]
n_desc_nmf_comp = 5
desc_nmf_cols = ["desc_nmf_"+str(i)for i in range(n_desc_nmf_comp)]
img_num_cols_all = ["P_RGB", "P_Dom_Px_Frac", "P_Dom_Px_Frac_Mean", "P_Dom_Score", "P_Dom_Score_Mean", "P_Vertex_X", "P_Vertex_Y",
"P_Bound_Conf", "P_Bound_Conf_Mean", "P_Bound_Imp_Frac", "P_Bound_Imp_Frac_Mean", "P_Label_Score", "P_Label_Score_Mean"]
img_num_cols_1 = ["Vertex_X_1", "Vertex_Y_1", "Bound_Conf_1", "Bound_Imp_Frac_1",
"Dom_Blue_1", "Dom_Green_1", "Dom_Red_1",
"RGBint_1", "Dom_Px_Fr_1", "Dom_Scr_1", "Lbl_Scr_1",]
img_num_cols_2 = ["Vertex_X_2", "Vertex_Y_2", "Bound_Conf_2", "Bound_Imp_Frac_2",
"Dom_Blue_2", "Dom_Green_2", "Dom_Red_2",
"RGBint_2", "Dom_Px_Fr_2", "Dom_Scr_2", "Lbl_Scr_2"]
img_num_cols_3 = ["Vertex_X_3", "Vertex_Y_3", "Bound_Conf_3", "Bound_Imp_Frac_3",
"Dom_Blue_3", "Dom_Green_3", "Dom_Red_3",
"RGBint_3", "Dom_Px_Fr_3", "Dom_Scr_3", "Lbl_Scr_3"]
img_lbl_cols_1 = ["Lbl_Img_1"]
img_lbl_cols_2 = ["Lbl_Img_2"]
img_lbl_cols_3 = ["Lbl_Img_3"]
img_lbl_col = ["Lbl_Dsc"]
n_iann_svd_comp = 5
iann_svd_cols = ["iann_svd_" + str(i)for i in range(n_iann_svd_comp)]
n_iann_nmf_comp = 5
iann_nmf_cols = ["iann_nmf_" + str(i)for i in range(n_iann_nmf_comp)]
n_entities_svd_comp = 5
entities_svd_cols = ["entities_svd_" + str(i)for i in range(n_entities_svd_comp)]
n_entities_nmf_comp = 5
entities_nmf_cols = ["entities_nmf_" + str(i)for i in range(n_entities_nmf_comp)]
item_cnt_incols = ["RescuerID", "Breed1", "Breed2", "Breed_Merge", "Age"]
item_cnt_cols = [c + "_Cnt" for c in item_cnt_incols]
item_cnt_mtype_cols = [c + "_Cnt_MType" for c in item_cnt_incols]
item_type_incols = ["RescuerID_Cnt", "Breed1_Cnt", "Breed2_Cnt", "Breed_Merge_Cnt", "Age_Cnt"]
item_type_cols = [c + "_StdType" for c in item_type_incols]
kbin_incols = ["Age", "Fee", "Fee_Per_Pet"] + item_cnt_cols
kbin_cols = [c + "_Kbin" for c in kbin_incols]
item_adp_incols = item_type_cols + item_cnt_cols + kbin_cols + item_cnt_mtype_cols
item_adp_cols = [c + "_Adp" for c in item_adp_incols]
fee_mean_incols = ["Breed1", "Breed2", "Age", "Breed_Merge", "State"] + item_cnt_cols + item_cnt_mtype_cols + item_type_cols
fee_mean_cols = ["Fee_Per_Pet_" + c for c in fee_mean_incols]
loo_incols = ["Pet_Breed", "State"]
loo_cols = [c + "_Loo" for c in loo_incols]
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
scaling_cols = ["Age", "Fee", "RescuerID_Cnt", "Breed1_Cnt", "Breed2_Cnt", "Breed_Merge_Cnt", "Age_Cnt", "Fee_Per_Pet"] + fee_mean_cols
ft_cat_cols = ["Breed1", "Breed2", "Breed_Merge", "Overall_Status"] + item_cnt_cols + item_type_cols + kbin_cols
ft_new_cols = ["Age", "Fee", "Quantity", "VideoAmt", "PhotoAmt"]
agg_calc = ["std", "skew", "mean", "max", "min"]
def feature_cols(fcc, fnc, agg_calc):
tmp_ft_cols = []
for cc in fcc:
for a in agg_calc:
if a != "COUNT":
for x in fnc:
tmp_ft_cols.append(cc + "_" + a + "(Pets." + x + ")")
 else:
tmp_ft_cols.append(cc + "_" + a + "(Pets)")
 return tmp_ft_cols
ft_cols = feature_cols(ft_cat_cols, ft_new_cols, agg_calc)
 barplot_cols = ind_num_cat_columns + item_type_cols + kbin_cols + item_cnt_mtype_cols
boxplot_cols = ind_cont_columns + desc_svd_cols + desc_nmf_cols + img_num_cols_all + img_num_cols_1 + img_num_cols_2 + img_num_cols_3 + img_lbl_cols_1 + img_lbl_cols_2 + img_lbl_cols_3 \
+ iann_svd_cols + iann_nmf_cols + entities_svd_cols + entities_nmf_cols + ft_cols + item_cnt_cols + item_adp_cols + fee_mean_cols
plog("Done")
<prepare_x_and_y>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )
def leaveoneout(X, y):
ret = pd.DataFrame(columns=Columns.loo_cols.value, index=None)
 R = 10
sums = np.zeros([len(X), 1])
 lens = np.zeros([len(X), 1])
 randomness =1+np.random.normal(scale = 0.03, size=len(sums)).reshape(len(sums), 1)
 for c in X.columns.values:
for uval in X[c].unique() :
indexes = X[X[c] == uval].index
s_tot = y.loc[indexes].sum()
l = len(indexes)-1+R
lens[indexes, 0] = 1/l
for i in indexes:
s_ind = s_tot - int(y.loc[i])
 sums[i,0] = s_ind
ret[c+"_Loo"] =(sums*lens*randomness ).ravel().tolist()
return ret
if 1 == 0:
for c in Columns.loo_cols.value:
if c in train_df.columns.values.tolist() :
plog("Dropping "+c +" in train dataset")
 train_df.drop(c, axis=1, inplace=True)
 plog("Dropped "+c +" in train dataset")
 df_loo = leaveoneout(train_df[Columns.loo_incols.value], train_df["AdoptionSpeed"])
 train_df = pd.concat([train_df,df_loo], axis=1)
 for c in Columns.loo_cols.value:
if c in test_df.columns.values.tolist() :
plog("Dropping "+c +" in test dataset")
 test_df.drop(c, axis=1, inplace=True)
 plog("Dropped "+c +" in test dataset")
 for c in Columns.loo_incols.value:
df_loo = train_df[[c,"AdoptionSpeed"]].groupby(c ).agg({'AdoptionSpeed':['sum', 'count']})
 df_loo.columns = df_loo.columns.droplevel(0)
 df_loo[c+"_Loo"] = df_loo["sum"] /(df_loo["count"]+10)
 df_loo.drop(["sum","count"], axis=1, inplace=True)
 test_df = test_df.set_index(c ).join(df_loo ).reset_index()
for c in Columns.loo_incols.value:
if train_df[c].isna().any() :
print("null", c)
 plog("Done")
<categorify>
target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()
train = train_features.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
len(target_cols )
def onehotencoding(df):
enc = OneHotEncoder(categories="auto")
 encodedcols = []
for c in ["Type", "Breed1", "Breed2", "Gender"]:
y = enc.fit_transform(df[[c]])
 col_name = enc.get_feature_names()
col_name = np_f.replace(col_name.astype(str), 'x0', c)
 dfo = pd.DataFrame(columns=col_name, data=y.toarray())
 df = pd.concat([df, dfo], axis=1)
 for ec in col_name:
encodedcols.append(ec)
 return df, encodedcols
if 1 == 0:
plog("One hot encoding started for train set")
 train_df, train_encodedcols = onehotencoding(train_df)
 plog("One hot encoding ended for train set")
 plog("One hot encoding started for test set")
 test_df, test_encodedcols = onehotencoding(test_df)
 plog("One hot encoding ended for test set")
 if train_encodedcols == test_encodedcols:
plog("Columns are same on test and train sets for one hot encoded cols")
 ohc_cols = train_encodedcols
plog("Done" )<feature_engineering>
if 1 == 0:
breed_id = {}
for id,name in zip(breed_labels.BreedID,breed_labels.BreedName):
breed_id[id] = name
cat_atts = breed_attributes['cat_breeds']
dog_atts = breed_attributes['dog_breeds']
cat_names = [i for i in cat_atts.keys() ]
dog_names = [i for i in dog_atts.keys() ]
def get_breed_atts(df, breed):
i=1
df_breed_ids = df[breed].unique()
print(i, len(df_breed_ids))
for id in df_breed_ids:
if id in breed_id.keys() :
name = breed_id[id]
if name in cat_names:
for key in cat_atts[name].keys() :
k = breed+"_"+key.strip().replace(" ", "_")
 if k not in df.columns.values.tolist() :
df[k] = np.NAN
indexes = df.loc[:,breed] == id
df.loc[indexes, k] = cat_atts[name][key]
if name in dog_names:
for key in dog_atts[name].keys() :
k = breed+"_"+key.strip().replace(" ", "_")
 if k not in df.columns.values.tolist() :
df[k] = np.NAN
indexes = df.loc[:,breed] == id
df.loc[indexes, k] = dog_atts[name][key]
i += 1
return df
plog("Getting Breed1 attributes for train")
 train_df = get_breed_atts(train_df, "Breed1")
 plog("Getted Breed1 attributes for train")
 plog("Getting Breed2 attributes for train")
 train_df = get_breed_atts(train_df, "Breed2")
 plog("Getted Breed2 attributes for train")
 print(train_df.info())
 sys.exit()
plog("Getting Breed1 attributes for test")
 test_df = get_breed_atts(test_df, "Breed1")
 plog("Getted Breed1 attributes for test")
 plog("Getting Breed2 attributes for test")
 test_df = get_breed_atts(test_df, "Breed2")
 plog("Getted Breed2 attributes for test")
<create_dataframe>
predictions = [sub6, sub10]
blend1 = pd.DataFrame()
blend1['sig_id'] = predictions[0]['sig_id']
for column in tqdm(target_cols):
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 blend1[column] = column_data[0] * 0.4 + column_data[1] * 0.6
print(blend1.shape)
 blend1.head()
def get_desc_anly(type, recalc):
if recalc == 1:
if type == "train":
path = ".. /input/train_sentiment/"
elif type == "test":
path = ".. /input/test_sentiment/"
plog("Getting description sentiment analysis for "+type+"_sentiment.csv")
 files = [f for f in os.listdir(path)if(f.endswith('.json')& os.path.isfile(path+f)) ]
df = pd.DataFrame(columns=["PetID", "DescScore", "DescMagnitude"])
 i = 0
for f in files:
with open(path+f, encoding="utf8")as json_data:
data = json.load(json_data)
 df.loc[i]= [f[:-5],data["documentSentiment"]["score"],data["documentSentiment"]["magnitude"]]
i = i+1
elif recalc == 0:
df = pd.read_csv(type+"_sentiment.csv")
 plog("Got description sentiment analysis for "+type+"_sentiment.csv")
 return df
rc = 1
if 1 == 0:
train_snt = get_desc_anly("train",rc)
 test_snt = get_desc_anly("test", rc)
 train_df = train_df.set_index("PetID" ).join(train_snt.set_index("PetID")).reset_index()
test_df = test_df.set_index("PetID" ).join(test_snt.set_index("PetID")).reset_index()
train_df["Description"].fillna("", inplace=True)
 train_df["Name"].fillna("", inplace=True)
 test_df["Description"].fillna("", inplace=True)
 test_df["Name"].fillna("", inplace=True)
 train_df["DescLength"] = train_df["Description"].str.len()
test_df["DescLength"] = test_df["Description"].str.len()
train_df["NameLength"] = train_df["Name"].str.len()
test_df["NameLength"] = test_df["Name"].str.len()
plog("Done" )<create_dataframe>
predictions = [blend1, sub9, sub7, sub8]
test_result = pd.DataFrame()
test_result['sig_id'] = predictions[0]['sig_id']
for column in tqdm(target_cols):
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 test_result[column] = column_data[0] * 0.2 + column_data[1] * 0.2 + column_data[2] * 0.4 + column_data[3] * 0.2
print(test_result.shape)
 test_result.head()
<create_dataframe><EOS> 
y_pred = test_result.copy()
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_cols)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_cols].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T