Kaggle competition: Mechanisms of Action (MoA) Prediction
gc.collect()<define_search_space>
NB = '101'
IS_TRAIN = False
MODEL_DIR = ".. /input/503-203-tabnet-with-nonscored-features-train/model"
INT_DIR = "interim"
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
lr=np.array([1e-4,1e-3,1e-2] )<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )<set_options>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
gc.collect()<feature_engineering>
n_comp = 50
n_dim = 15
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_g.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_g.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_g.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_g.pkl")
 data2 = pca.transform(data[GENES])
 data3 = umap.transform(data[GENES])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
 n_comp = 15
n_dim = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_c.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_c.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_c.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_c.pkl")
 data2 = pca.transform(data[CELLS])
 data3 = umap.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0 )<compute_test_metric>
if IS_TRAIN:
var_thresh = VarianceThreshold(threshold=0.5 ).fit(train_features.iloc[:, 4:])
 pd.to_pickle(var_thresh, f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 else:
var_thresh = pd.read_pickle(f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 data = train_features.append(test_features)
 data_transformed = var_thresh.transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print(train_features.shape)
 print(test_features.shape )
accuracy_np(probs, y )<prepare_output>
train = train_features[train_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
preds = np.argmax(probs, axis=1)
 probs = probs[:,1]<predict_on_test>
train.to_pickle(f"{INT_DIR}/{NB}_train_preprocessed.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_preprocessed.pkl" )
temp = learn.predict(is_test=True )<prepare_output>
NB = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
HIDDEN_SIZE = 2048
EPOCHS = 15
BATCH_SIZE = 2048
NFOLDS = 10
NREPEATS = 1
NSEEDS = 5
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2
pred_test = np.argmax(temp, axis=1 )<prepare_output>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def calc_valid_log_loss(train, target, target_cols):
y_pred = train[target_cols].values
y_true = target[target_cols].values
y_true_t = torch.from_numpy(y_true.astype(np.float64)).clone()
y_pred_t = torch.from_numpy(y_pred.astype(np.float64)).clone()
return torch.nn.BCELoss()(y_pred_t, y_true_t ).to('cpu' ).detach().numpy().copy()
probs = np.exp(temp[:,1] )<create_dataframe>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=HIDDEN_SIZE):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
submission = pd.DataFrame({'id':os.listdir(f'{PATH}test'), 'label':probs} )<feature_engineering>
def run_training(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=PCT_START, div_factor=DIV_FACS,
max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
best_loss_epoch = -1
if IS_TRAIN:
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if valid_loss < best_loss:
best_loss = valid_loss
best_loss_epoch = epoch
oof[val_idx] = valid_preds
model.to('cpu')
 torch.save(model.state_dict() , f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth")
 model.to(DEVICE)
 if epoch % 10 == 0 or epoch == EPOCHS-1:
print(f"seed: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}, best_loss: {best_loss:.6f}, best_loss_epoch: {best_loss_epoch}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.load_state_dict(torch.load(f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth"))
model.to(DEVICE)
 if not IS_TRAIN:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
submission['id'] = submission['id'].map(lambda x: x.split('.')[0] )<data_type_conversions>
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = RepeatedMultilabelStratifiedKFold(n_splits=NFOLDS, n_repeats=NREPEATS, random_state=None)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
submission['id'] = submission['id'].astype(int )<sort_values>
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions
submission = submission.sort_values('id' )<save_to_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
submission.to_csv('.. /working/output.csv', index=False )<set_options>
train = pd.read_pickle(f"{INT_DIR}/101_train_preprocessed.pkl")
 test = pd.read_pickle(f"{INT_DIR}/101_test_preprocessed.pkl" )
%matplotlib inline
plt.style.use('ggplot')
 warnings.filterwarnings("ignore" )<load_from_csv>
train_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_train_preprocessed.pkl")
 test_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_test_preprocessed.pkl" )
breeds = pd.read_csv('.. /input/breed_labels.csv')
 colors = pd.read_csv('.. /input/color_labels.csv')
 states = pd.read_csv('.. /input/state_labels.csv')
 train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sub = pd.read_csv('.. /input/test/sample_submission.csv')
 train['DatasetType'] = 'train'
test['DatasetType'] = 'test'
all_data = pd.concat([train, test] )<import_modules>
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
print(train_targets_nonscored.shape)
 train = train.merge(train_targets_nonscored, on='sig_id' )
import sklearn
from sklearn.preprocessing import scale
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn import preprocessing<categorify>
run_seeds(train, test, feature_cols, target_cols )
enc = OneHotEncoder(handle_unknown='ignore')
 l = LogisticRegression()<prepare_x_and_y>
valid_loss_total = calc_valid_log_loss(train, target, target_cols)
 print(f"CV loss: {valid_loss_total}" )
features = train.columns.difference(["Name", "Description", "PetID", "RescuerID", "AdoptionSpeed"])
 train_x = train[features]
train_y = train["AdoptionSpeed"]
test_x = test[features]<train_model>
train.to_pickle(f"{INT_DIR}/{NB}_train_nonscored_pred.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_nonscored_pred.pkl" )
enc.fit(train_x)
 x_train_1h = enc.transform(train_x)
 train_xx,cv_x,train_yy,cv_y=train_test_split(x_train_1h,train_y,test_size=0.2)
 l.fit(train_xx,list(train_yy))
y_pred = l.predict(cv_x)
 accuracy = accuracy_score(list(cv_y),y_pred)
 accuracy<predict_on_test>
valid_results = train_targets_nonscored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_nonscored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
enc.fit(train_x)
 X_train_one_hot = enc.transform(train_x)
 X_test_one_hot = enc.transform(test_x)
 l.fit(X_train_one_hot,train_y)
 y_pred = l.predict(X_test_one_hot)
 print(X_train_one_hot.shape )<data_type_conversions>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
for i,val in enumerate(y_pred):
sub.at[i,'AdoptionSpeed'] = val
sub.AdoptionSpeed = sub.AdoptionSpeed.astype(int)
 sub.head()<save_to_csv>
from pytorch_tabnet.tab_model import TabNetRegressor
sub.to_csv('submission.csv', index=False )<save_to_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
sub.to_csv('submission.csv', index=False )<import_modules>
NB = '503'
NB_PREV = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
NFOLDS = 10
NREPEATS = 1
NSEEDS = 3
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic')
 def warn(*args, **kwargs):
pass
warnings.warn = warn
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128<set_options>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
plt.rcParams['figure.figsize'] =(12, 9)
<load_from_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<load_from_csv>
use_test_for_preprocessing = False
for col in(GENES + CELLS):
if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 if use_test_for_preprocessing:
raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 transformer.fit(raw_vec)
 else:
raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<merge>
n_comp = 90
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[GENES])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 data2 =(fa.transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[CELLS])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 data2 =(fa.transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
def maek_features(src_df):
rescuer_count = src_df.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_CNT']
src_df = src_df.merge(rescuer_count, how='left', on='RescuerID')
 return src_df<feature_engineering>
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 if IS_TRAIN:
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 pd.to_pickle(kmeans, f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 else:
kmeans = pd.read_pickle(f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 train[f'clusters_{kind}'] = kmeans.predict(train_)
 test[f'clusters_{kind}'] = kmeans.predict(test_)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )
train = maek_features(train)
 test = maek_features(test )<load_from_csv>
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )
train_img = pd.read_csv(".. /input/pretrainednn/train_img_features.csv")
 test_img = pd.read_csv(".. /input/pretrainednn/test_img_features.csv")
 train_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 test_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 train = pd.concat([train, train_img], axis=1)
 test = pd.concat([test, test_img], axis=1 )<load_pretrained>
remove_vehicle = True
if remove_vehicle:
trt_idx = train_features['cp_type']=='trt_cp'
train_features = train_features.loc[trt_idx].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[trt_idx].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[trt_idx].reset_index(drop=True)
 else:
pass
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f )<define_variables>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']<feature_engineering>
train_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_train_nonscored_pred.pkl')
 test_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_test_nonscored_pred.pkl' )
breed_id = {}
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = name<define_variables>
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]<feature_engineering>
train = train.merge(train_nonscored_pred[train_targets_nonscored.columns], on='sig_id')
 test = test.merge(test_nonscored_pred[train_targets_nonscored.columns], on='sig_id' )
for id in train['Breed1']:
if id in breed_id.keys() :
name = breed_id[id]
if name in breed_names_1:
for key in cat_ratings[name].keys() :
train[key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
train[key] = dog_ratings[name][key]<define_variables>
nonscored_target = [c for c in train_targets_nonscored.columns if c != "sig_id"]
for col in(nonscored_target):
vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )<count_unique_values>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
len(feature_cols )
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )<load_pretrained>
num_features=len(feature_cols)
 num_targets=len(target_cols )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()
class SmoothedLogLossMetric(Metric):

 def __init__(self, smoothing=0.001):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0)
 class LogLossMetric(Metric):

 def __init__(self, smoothing=0.0):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0 )
aggregates = ['mean', 'sum', 'var']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
def process_data(data):
data.loc[:, 'cp_time'] = data.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2, 0: 0, 1: 1, 2: 2})
 data.loc[:, 'cp_dose'] = data.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1, 0: 0, 1: 1})
 return data
def run_training_tabnet(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed, filename="tabnet"):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 loss_fn = LabelSmoothing(0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
if IS_TRAIN:
model.fit(X_train=x_train, y_train=y_train,
eval_set=[(x_valid, y_valid)], eval_metric=[LogLossMetric, SmoothedLogLossMetric],
max_epochs=200, patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=loss_fn 
)
 model.save_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}")
 x_test = test_[feature_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 model.load_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}.model")
 valid_preds = model.predict(x_valid)
 valid_preds = torch.sigmoid(torch.as_tensor(valid_preds)).detach().cpu().numpy()
oof[val_idx] = valid_preds
predictions = model.predict(x_test)
 predictions = torch.sigmoid(torch.as_tensor(predictions)).detach().cpu().numpy()
return oof, predictions
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training_tabnet(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
train.to_pickle(f"{INT_DIR}/{NB}_pre_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_pre_test.pkl" )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))<define_variables>
run_seeds(train, test, feature_cols, target_cols, NFOLDS, NSEEDS )
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))<define_variables>
train.to_pickle(f"{INT_DIR}/{NB}_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test.pkl" )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID'] 
<merge>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<normalization>
sub6 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub6.to_csv('submission.csv', index=False )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<data_type_conversions>
import glob
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )<feature_engineering>
!python.. /input/inference-best-lb/deepinsight-resnest-lightning-v2-inference.py
sub5 = pd.read_csv('submission_resnest_v2.csv' )
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<count_missing_values>
!python.. /input/inference-best-lb/deepinsight-efficientnet-lightning-v7-b3-inference.py
sub4 = pd.read_csv('./submission_effnet_v7_b3.csv' )
np.sum(pd.isnull(X_train))<count_missing_values>
! python.. /input/inference-best-lb/simple-nn-new-split-inference.py
sub3 = pd.read_csv('./submission.csv' )
np.sum(pd.isnull(X_test))<import_modules>
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 !python.. /input/inference-best-lb/2heads-ResNest-inference.py
sub2 = pd.read_csv('./submission.csv' )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<init_hyperparams>
!python.. /input/inference-best-lb/3stage-nn-inference.py
sub1 = pd.read_csv('./submission_2stageNN_with_ns_oldcv_0.01822.csv' )
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 10000
n_splits = 5<drop_column>
!python.. /input/inference-best-lb/simple-nn-old-split-inference.py
sub7 = pd.read_csv('submission_script_simpleNN_oldcv_0.01836.csv' )
X_train = X_train.drop('img_Unnamed: 0',axis=1)
 X_test = X_test.drop('img_Unnamed: 0',axis=1 )<prepare_x_and_y>
submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission.iloc[:, 1:] = 0
submission.iloc[:, 1:] =(sub1.iloc[:,1:]*0.37 + sub3.iloc[:,1:]*0.1 + sub4.iloc[:,1:]*0.18 +sub5.iloc[:,1:]*0.15)*0.9 + sub6.iloc[:,1:]*0.1 + sub7.iloc[:,1:]*0.09 + sub2.iloc[:,1:]*0.09
submission.to_csv('submission.csv', index=False )