Kaggle competition: Mechanisms of Action (MoA) Prediction
lgbm_train = lgbm.Dataset(X_train, y_train, categorical_feature=category_columns)
 lgbm_valid = lgbm.Dataset(X_val, y_val, categorical_feature=category_columns )<train_model>
! pip install.. /input/moa-pytorch16/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
model_lgbm = lgbm.train(params_lgbm, lgbm_train, 50000, valid_sets=[lgbm_valid], verbose_eval= 5000, categorical_feature=category_columns, early_stopping_rounds = 500 )<predict_on_test>
!pip install.. /input/moaiterative/iterative_stratification-0.1.6-py3-none-any.whl
print(( np.argmax(model_lgbm.predict(X_train), axis=1)== y_train ).sum() / y_train.shape[0])
 print(( np.argmax(model_lgbm.predict(X_val), axis=1)== y_val ).sum() / y_val.shape[0] )<predict_on_test>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
val_lgbm = model_lgbm.predict(X_val)
 val_gb = model_gb.predict_log_proba(X_val)
 val_mixed =(val_gb + val_lgbm)/ 2 
(np.argmax(val_mixed, axis=1)== y_val ).sum() / y_val.shape[0] 
<save_to_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(1984 )
test_pred = model_xgb.predict(df_test)
 test_petid = pd.DataFrame(test_petid)
 submission = test_petid.join(pd.DataFrame(test_pred, columns=['AdoptionSpeed']))
submission.to_csv('submission.csv', index=False )<feature_engineering>
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
train_features = train.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train
kappa_scorer = None
os.environ['USER'] = 'root'
os.system('pip install.. /input/xlearn/xlearn/xlearn-0.40a1/')
 print(os.listdir(".. /input"))
print(os.listdir(".. /input/petfinder-adoption-prediction/"))
print(os.listdir(".. /input/petfinder-adoption-prediction/train"))
print(os.listdir(".. /input/petfinder-adoption-prediction/test"))<load_from_csv>
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored )
train_df = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")
 test_df = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv" )<prepare_x_and_y>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
y = train_df["AdoptionSpeed"]<load_from_csv>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def load_data(train_path, cols):
if isinstance(train_path, list):
df_base = pd.read_csv(train_path[0])
 for i in range(1, len(train_path)) :
df_base.append(pd.read_csv(train_path[i]))
df_base = df_base[cols]
else:
df = pd.read_csv(train_path)
 df_base = df[cols]
x = df_base.values.reshape(-1, df_base.shape[1] ).astype('float32')
 standardizer = preprocessing.StandardScaler()
x = standardizer.fit_transform(x)
 return x, standardizer
def numpyToTensor(x):
x_train = torch.from_numpy(x ).to(device)
 return x_train
class DataBuilder(Dataset):
def __init__(self, path, cols):
self.x, self.standardizer = load_data(path, cols)
 self.x = numpyToTensor(self.x)
 self.len=self.x.shape[0]
def __getitem__(self,index):
return self.x[index]
def __len__(self):
return self.len
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv' )<load_pretrained>
class Autoencoder(nn.Module):
def __init__(self,D_in, H, H2, latent_dim):
super(Autoencoder,self ).__init__()
self.linear1=nn.Linear(D_in,H)
 self.lin_bn1 = nn.BatchNorm1d(num_features=H)
 self.linear2=nn.Linear(H,H2)
 self.lin_bn2 = nn.BatchNorm1d(num_features=H2)
 self.linear3=nn.Linear(H2,H2)
 self.lin_bn3 = nn.BatchNorm1d(num_features=H2)
 self.fc1 = nn.Linear(H2, latent_dim)
 self.bn1 = nn.BatchNorm1d(num_features=latent_dim)
 self.fc21 = nn.Linear(latent_dim, latent_dim)
 self.fc22 = nn.Linear(latent_dim, latent_dim)
 self.fc3 = nn.Linear(latent_dim, latent_dim)
 self.fc_bn3 = nn.BatchNorm1d(latent_dim)
 self.fc4 = nn.Linear(latent_dim, H2)
 self.fc_bn4 = nn.BatchNorm1d(H2)
 self.linear4=nn.Linear(H2,H2)
 self.lin_bn4 = nn.BatchNorm1d(num_features=H2)
 self.linear5=nn.Linear(H2,H)
 self.lin_bn5 = nn.BatchNorm1d(num_features=H)
 self.linear6=nn.Linear(H,D_in)
 self.lin_bn6 = nn.BatchNorm1d(num_features=D_in)
 self.leaky_relu = nn.LeakyReLU()
def encode(self, x):
lin1 = self.leaky_relu(self.lin_bn1(self.linear1(x)))
 lin2 = self.leaky_relu(self.lin_bn2(self.linear2(lin1)))
 lin3 = self.leaky_relu(self.lin_bn3(self.linear3(lin2)))
 fc1 = F.leaky_relu(self.bn1(self.fc1(lin3)))
 r1 = self.fc21(fc1)
 r2 = self.fc22(fc1)
 return r1, r2
def reparameterize(self, mu, logvar):
if self.training:
std = logvar.mul(0.5 ).exp_()
eps = Variable(std.data.new(std.size() ).normal_())
 return eps.mul(std ).add_(mu)
 else:
return mu
def decode(self, z):
fc3 = self.leaky_relu(self.fc_bn3(self.fc3(z)))
 fc4 = self.leaky_relu(self.fc_bn4(self.fc4(fc3)))
 lin4 = self.leaky_relu(self.lin_bn4(self.linear4(fc4)))
 lin5 = self.leaky_relu(self.lin_bn5(self.linear5(lin4)))
 return self.lin_bn6(self.linear6(lin5))
def forward(self, x):
mu, logvar = self.encode(x)
 z = self.reparameterize(mu, logvar)
 return self.decode(z), mu, logvar
class customLoss(nn.Module):
def __init__(self):
super(customLoss, self ).__init__()
self.mse_loss = nn.MSELoss(reduction="sum")
 def forward(self, x_recon, x, mu, logvar):
loss_MSE = self.mse_loss(x_recon, x)
 loss_KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2)- logvar.exp())
 return loss_MSE + loss_KLD
def weights_init_uniform_rule(m):
classname = m.__class__.__name__
if classname.find('Linear')!= -1:
n = m.in_features
y = 1.0/np.sqrt(n)
 m.weight.data.uniform_(-y, y)
 m.bias.data.fill_(0)
 def train(epoch, dataloader, model, optimizer):
loss_mse = customLoss()
model.train()
train_loss = 0
for batch_idx, data in enumerate(dataloader):
data = data.to(device)
 optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
 loss = loss_mse(recon_batch, data, mu, logvar)
 loss.backward()
train_loss += loss.item()
optimizer.step()
if epoch % 100 == 0:
print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(dataloader.dataset)))
 def encode_features(data_loader, model, optimizer):
mu_output = []
logvar_output = []
with torch.no_grad() :
for i,(data)in enumerate(data_loader):
data = data.to(device)
 optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
 mu_tensor = mu
mu_output.append(mu_tensor)
 mu_result = torch.cat(mu_output, dim=0)
 logvar_tensor = logvar
logvar_output.append(logvar_tensor)
 logvar_result = torch.cat(logvar_output, dim=0)
 return mu_result
def get_encoder_features(TRAIN_DATA_PATH, TEST_DATA_PATH, cols, latent_dim, name, train_flag=True):
print('Reading train data...')
 data_set_train=DataBuilder(TRAIN_DATA_PATH, cols)
 trainloader=DataLoader(dataset=data_set_train,batch_size=512)
 print('Reading test data...')
 data_set_test=DataBuilder(TEST_DATA_PATH, cols)
 testloader=DataLoader(dataset=data_set_test,batch_size=512)
 print('Reading data...')
 data_set = DataBuilder([TRAIN_DATA_PATH, TEST_DATA_PATH], cols)
 loader=DataLoader(dataset=data_set,batch_size=512)
 print('Prepare params...')
 D_in = data_set_train.x.shape[1]
H = D_in -(D_in - latent_dim)// 4
H2 = D_in -(D_in - latent_dim)// 2
model = Autoencoder(D_in, H, H2, latent_dim ).to(device)
 model.apply(weights_init_uniform_rule)
 optimizer = optim.Adam(model.parameters() , lr=1e-3)
 loss_mse = customLoss()
epochs = 500
log_interval = 50
val_losses = []
train_losses = []
print("train or load model...")
 if train_flag:
print("train model...")
 for epoch in range(1, epochs + 1):
train(epoch, loader, model, optimizer)
 torch.save(model.state_dict() , name)
 else:
print("load model...")
 model.load_state_dict(torch.load(name))
model.eval()
print("extract features...")
 train_encoder = encode_features(trainloader, model, optimizer)
 test_encoder = encode_features(testloader, model, optimizer)
 return train_encoder, test_encoder
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f)
 cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
breed_id = {}<define_variables>
TRAIN_DATA_PATH = '.. /input/lish-moa/train_features.csv'
TEST_DATA_PATH = '.. /input/lish-moa/test_features.csv'
gene_encode_dim = 50
cell_encode_dim = 30
genes_model_name = '.. /input/moa-tabnet-1830/autoencode_genes_50_30_default'
cells_model_name = '.. /input/moa-tabnet-1830/autoencode_cells_50_30_default'
train_gene_encoder, test_gene_encoder = get_encoder_features(TRAIN_DATA_PATH, TEST_DATA_PATH, GENES, gene_encode_dim, genes_model_name,
train_flag=False)
 train_cell_encoder, test_cell_encoder = get_encoder_features(TRAIN_DATA_PATH, TEST_DATA_PATH, CELLS, cell_encode_dim, cells_model_name,
train_flag=False)
 train_gene_encoder_pd = pd.DataFrame(train_gene_encoder.cpu().numpy() , columns=[f'VAE_G_{i}' for i in range(gene_encode_dim)])
 test_gene_encoder_pd = pd.DataFrame(test_gene_encoder.cpu().numpy() , columns=[f'VAE_G_{i}' for i in range(gene_encode_dim)])
 train_cell_encoder_pd = pd.DataFrame(train_cell_encoder.cpu().numpy() , columns=[f'VAE_C_{i}' for i in range(cell_encode_dim)])
 test_cell_encoder_pd = pd.DataFrame(test_cell_encoder.cpu().numpy() , columns=[f'VAE_C_{i}' for i in range(cell_encode_dim)] )
pet_attributes = set(list(cat_ratings["Abyssinian"].keys())+ list(dog_ratings["German Shepherd Dog"].keys()))<feature_engineering>
train_gene_encoder_pd['sig_id'] = train_features['sig_id']
train_cell_encoder_pd['sig_id'] = train_features['sig_id']
test_gene_encoder_pd['sig_id'] = test_features['sig_id']
test_cell_encoder_pd['sig_id'] = test_features['sig_id']
train_features = train_features.merge(train_gene_encoder_pd, on='sig_id')
 train_features = train_features.merge(train_cell_encoder_pd, on='sig_id')
 test_features = test_features.merge(test_gene_encoder_pd, on='sig_id')
 test_features = test_features.merge(test_cell_encoder_pd, on='sig_id' )
for i in pet_attributes:
train_df[i] = 0
test_df[i] = 0<feature_engineering>
n_comp = 80
pca_g = load('.. /input/moa-tabnet-1830/pca_g.joblib')
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 = pca_g.transform(data[GENES])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 40
pca_c = load('.. /input/moa-tabnet-1830/pca_c.joblib')
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 = pca_c.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_1_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_1_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 80
print("FastICA on genes")
 transformer = load('.. /input/moa-tabnet-1830/FastICA_g.joblib')
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 = transformer.transform(data[GENES])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FastICA_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FastICA_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 40
print("FastICA on cells")
 transformer = load('.. /input/moa-tabnet-1830/FastICA_c.joblib')
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 = transformer.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FastICA_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FastICA_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 80
print("FactorAnalysis on genes")
 transformer = load('.. /input/moa-tabnet-1830/FactorAnalysis_g.joblib')
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 = transformer.transform(data[GENES])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FactorAnalysis_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FactorAnalysis_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 40
print("FactorAnalysis on cells")
 transformer = load('.. /input/moa-tabnet-1830/FactorAnalysis_c.joblib')
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 = transformer.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FactorAnalysis_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FactorAnalysis_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 80
print("FeatureAgglomeration on genes")
 transformer = load('.. /input/moa-tabnet-1830/FeatureAgglomeration_g.joblib')
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 = transformer.transform(data[GENES])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FeatureAgglomeration_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FeatureAgglomeration_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 40
print("FeatureAgglomeration on cells")
 transformer = load('.. /input/moa-tabnet-1830/FeatureAgglomeration_c.joblib')
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 = transformer.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'FeatureAgglomeration_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'FeatureAgglomeration_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = str(name ).lower()
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]
for i, id in enumerate(train_df['Breed1']):
if id in breed_id.keys() :
name = str(breed_id[id] ).lower()
if name in breed_names_1:
for key in cat_ratings[name].keys() :
train_df.loc[i, key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
train_df.loc[i, key] = dog_ratings[name][key]
for i, id in enumerate(test_df['Breed1']):
if id in breed_id.keys() :
str(breed_id[id] ).lower()
if name in breed_names_1:
for key in cat_ratings[name].keys() :
test_df.loc[i, key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
test_df.loc[i, key] = dog_ratings[name][key]<define_variables>
def stat_feat(df, features_g, features_c):
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return df
train_features = stat_feat(train_features, GENES, CELLS)
 test_features = stat_feat(test_features, GENES, CELLS )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )<count_unique_values>
class VarianceThreshold:
def __init__(self, threshold):
self.threshold = threshold
def fit(self, df, cont_cols):
self.cont_cols = cont_cols
self.var = df[cont_cols].var()
good_cols = self.var[self.var > self.threshold]
self.index = good_cols.index.to_list()
self.dropcols = [x for x in cont_cols if x not in self.var[self.var > self.threshold].index.to_list() ]
self.validcols = [x for x in cont_cols if x in self.var[self.var > self.threshold].index.to_list() ]
def transform(self, df):
return df.drop(self.dropcols, axis=1)
 def fit_transform(self, df, cont_cols):
self.fit(df, cont_cols)
 return self.transform(df), self.validcols
train_df_ids = train_df[['PetID']]
print(train_df_ids.shape)
 train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 print(len(train_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(train_imgs_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / train_df_ids.shape[0]))
train_df_ids = train_df[['PetID']]
train_df_metadata = pd.DataFrame(train_metadata_files)
 train_df_metadata.columns = ['metadata_filename']
train_metadata_pets = train_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 train_df_metadata = train_df_metadata.assign(PetID=train_metadata_pets)
 print(len(train_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(train_metadata_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / train_df_ids.shape[0]))
train_df_ids = train_df[['PetID']]
train_df_sentiment = pd.DataFrame(train_sentiment_files)
 train_df_sentiment.columns = ['sentiment_filename']
train_sentiment_pets = train_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 train_df_sentiment = train_df_sentiment.assign(PetID=train_sentiment_pets)
 print(len(train_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(train_sentiment_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / train_df_ids.shape[0]))<count_unique_values>
cont_cols = [col for col in train_features.columns.to_list() if col not in ['sig_id','cp_type','cp_time','cp_dose']]
test_df_ids = test_df[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test_df[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test_df[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )<load_pretrained>
threshold = 0.2
print('Variance Threshold:', threshold)
 VarThres = VarianceThreshold(threshold)
 train_features, cont_cols = VarThres.fit_transform(train_features, cont_cols)
 test_features = VarThres.transform(test_features )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r', encoding='utf-8')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r', encoding='utf-8')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<feature_engineering>
qt = load('.. /input/moa-tabnet-1830/qt.joblib')
 train_features[cont_cols] = qt.fit_transform(train_features[cont_cols])
 test_features[cont_cols] = qt.transform(test_features[cont_cols] )
aggregates = ['mean', 'sum', 'var', 'max', 'min']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32)
train_proc = train_df.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test_df.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
<concatenate>
def evals(model, X, y, verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.clamp(y_preds, 0.0,1.0 ).detach().numpy()
score = log_loss_multi(y, y_preds)
 return y_preds, score
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False, axis = 0)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))<create_dataframe>
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']<merge>
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 unique_count = X.groupby(['RescuerID'])['Breed1'].nunique().reset_index()
unique_count.columns = ['RescuerID', 'RescuerID_UNIQUE']
X_temp = X_temp.merge(unique_count, how='left', on='RescuerID')
 X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )<feature_engineering>
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
n_components = 16
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp.reset_index(drop = True), text_features], axis=1)
 text_columns = ['metadata_annots_top_desc', 'sentiment_entities']
for i in text_columns:
X_temp = X_temp.drop(i, axis=1 )<drop_column>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
train_df = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
test_df = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
test_df = test_df.drop(['AdoptionSpeed'], axis=1 )<define_variables>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
cat_cols = ['Breed1','Breed2', 'Gender', 'Color1', 'Color2', 'Color3', 'State', 'label_description']<define_variables>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 30
self.EPOCHS = 200
self.num_ensembling = 1
self.seed = 42
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = ".. /input/moa-tabnet-1830/raw_step1_dae_new/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()
num_cols = ['Fee', 'Age', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'VideoAmt', 'PhotoAmt',
'Type', 'doc_sent_mag', 'doc_sent_score', 'vertex_x', 'vertex_y',
'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green',
'dominant_red', 'label_score', 'state_gdp', 'state_population',
'sentiment_sentiment_magnitude_MEAN', 'sentiment_sentiment_magnitude_SUM',
'sentiment_sentiment_score_MEAN', 'sentiment_sentiment_score_SUM',
'sentiment_sentiment_document_magnitude_MEAN', 'sentiment_sentiment_document_magnitude_SUM',
'sentiment_sentiment_document_score_MEAN', 'sentiment_sentiment_document_score_SUM',
'metadata_metadata_annots_score_MEAN', 'metadata_metadata_annots_score_SUM', 'metadata_metadata_color_score_MEAN',
'metadata_metadata_color_score_SUM', 'metadata_metadata_color_pixelfrac_MEAN', 'metadata_metadata_color_pixelfrac_SUM',
'metadata_metadata_crop_conf_MEAN', 'metadata_metadata_crop_conf_SUM', 'metadata_metadata_crop_importance_MEAN',
'metadata_metadata_crop_importance_SUM', 'RescuerID_COUNT',
'agg_mean_Age', 'agg_mean_Gender', 'agg_mean_Color1',
'agg_mean_Color2', 'agg_mean_Color3', 'agg_mean_FurLength', 'agg_mean_Vaccinated', 'agg_mean_Dewormed',
'agg_mean_Sterilized', 'agg_mean_Health', 'agg_mean_Quantity', 'agg_mean_Fee', 'agg_mean_PhotoAmt',
'agg_mean_VideoAmt', 'agg_std_Age', 'agg_std_Gender', 'agg_std_Color1', 'agg_std_Color2',
'agg_std_Color3', 'agg_std_FurLength', 'agg_std_Vaccinated', 'agg_std_Dewormed', 'agg_std_Sterilized',
'agg_std_Health', 'agg_std_Quantity', 'agg_std_Fee', 'agg_std_PhotoAmt', 'agg_std_VideoAmt', 'name_len',
'average_word_length', 'num_words', 'desc_char_length', 'RescuerID_UNIQUE']<define_variables>
X_test = np.concatenate([cat_test, numerical_test], axis=1 )
text_cols = ['Description']<load_pretrained>
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
idx_all = []
for seed in range(cfg.num_ensembling):
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 idx_all.append(val_idx)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=64, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
train_id = train_df['PetID']
test_id = test_df['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_sentiment/' + pet + '.json', 'r', encoding = "utf-8")as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train_df.loc[:, 'doc_sent_mag'] = doc_sent_mag
train_df.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_sentiment/' + pet + '.json', 'r', encoding = "utf-8")as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test_df.loc[:, 'doc_sent_mag'] = doc_sent_mag
test_df.loc[:, 'doc_sent_score'] = doc_sent_score
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_metadata/' + pet + '-1.json', 'r', encoding = "utf-8")as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train_df.loc[:, 'vertex_x'] = vertex_xs
train_df.loc[:, 'vertex_y'] = vertex_ys
train_df.loc[:, 'bounding_confidence'] = bounding_confidences
train_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
train_df.loc[:, 'dominant_blue'] = dominant_blues
train_df.loc[:, 'dominant_green'] = dominant_greens
train_df.loc[:, 'dominant_red'] = dominant_reds
train_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train_df.loc[:, 'dominant_score'] = dominant_scores
train_df.loc[:, 'label_description'] = label_descriptions
train_df.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_metadata/' + pet + '-1.json', 'r', encoding = "utf-8")as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test_df.loc[:, 'vertex_x'] = vertex_xs
test_df.loc[:, 'vertex_y'] = vertex_ys
test_df.loc[:, 'bounding_confidence'] = bounding_confidences
test_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
test_df.loc[:, 'dominant_blue'] = dominant_blues
test_df.loc[:, 'dominant_green'] = dominant_greens
test_df.loc[:, 'dominant_red'] = dominant_reds
test_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test_df.loc[:, 'dominant_score'] = dominant_scores
test_df.loc[:, 'label_description'] = label_descriptions
test_df.loc[:, 'label_score'] = label_scores<import_modules>
idx_list = []
for idx_ in idx_all:
for ix in idx_:
idx_list.append(ix)
 oof_preds_df = pd.DataFrame(oof_preds_all[0])
 oof_preds_df['idx'] = idx_list
import tensorflow<import_modules>
train_org = pd.read_csv('.. /input/lish-moa/train_features.csv')
 target_org = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_org = train_org.loc[train_org['cp_type']=='trt_cp'].reset_index(drop=True)
 train_org['idx'] = train_org.index
oof_final = train_org.merge(oof_preds_df, how='left', on='idx')[list(range(206)) + ['sig_id']]
oof_final = target_org.merge(oof_final, how='left', on='sig_id')[list(range(206)) ].fillna(0.0 )
from tqdm import tqdm
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences<define_variables>
targets = target_org[[col for col in target_org.columns.to_list() if col != 'sig_id']]
log_loss_multi(targets.to_numpy() , oof_final.to_numpy() )
num_words = 200000
maxlen = 200
embed_size = 300<drop_column>
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
test_df = test_df.reset_index(drop = True )<load_from_csv>
submission[columns] = preds_test.mean(1 ).mean(0)
 submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
train_df.loc[:, "Description"] = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")["Description"]
test_df.loc[:, "Description"] = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv")["Description"]<data_type_conversions>
submission.to_csv("submission.csv", index=False )
<data_type_conversions><EOS> 
submission.to_csv("submission.csv", index=False )