Kaggle competition: Mechanisms of Action (MoA) Prediction
X_tr_fmix =enc.fit_transform(np.array(X_train['feature_mix'] ).reshape(-1,1))
X_cv_fmix =enc.transform(np.array(X_cv['feature_mix'] ).reshape(-1,1))
X_te_fmix =enc.transform(np.array(test['feature_mix'] ).reshape(-1,1))<concatenate>
! pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
X_tr = hstack(( X_tr_age,X_tr_fare,X_tr_sex,X_tr_pclass,X_tr_emb)).tocsr()
X_cv = hstack(( X_cv_age,X_cv_fare,X_cv_sex,X_cv_pclass,X_cv_emb)).tocsr()
X_te = hstack(( X_te_age,X_te_fare,X_te_sex,X_te_pclass,X_te_emb)).tocsr()
print(X_tr.shape)
 print(X_te.shape)
 print(X_cv.shape )<train_on_grid>
!pip install.. /input/iterative-stratification/iterative_stratification-0.1.6-py3-none-any.whl
alpha = [10 ** x for x in range(-5, 1)]
cv_log_error_array=[]
for i in alpha:
clf = SGDClassifier(alpha=i,class_weight="balanced", penalty='l2', loss='log', random_state=42)
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_cv)
 cv_log_error_array.append(log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
print('For values of alpha = ', i, "The log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
fig, ax = plt.subplots()
ax.plot(alpha, cv_log_error_array,c='g')
 for i, txt in enumerate(np.round(cv_log_error_array,3)) :
ax.annotate(( alpha[i],np.round(txt,3)) ,(alpha[i],cv_log_error_array[i]))
plt.grid()
plt.title("Cross Validation Error for each alpha")
 plt.xlabel("Alpha i's")
 plt.ylabel("Error measure")
 plt.show()
best_alpha = np.argmin(cv_log_error_array)
 clf = SGDClassifier(alpha=alpha[best_alpha],class_weight="balanced", penalty='l2', loss='log', random_state=42)
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_tr)
 print('For values of best alpha = ', alpha[best_alpha], "The train log loss is:",log_loss(y_train, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_cv)
 print('For values of best alpha = ', alpha[best_alpha], "The cross validation log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_te)
 print('For values of best alpha = ', alpha[best_alpha], "The test log loss is:",log_loss(y_test, predict_y, labels=clf.classes_, eps=1e-15))
lr=alpha[best_alpha]<save_to_csv>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
pred=sig_clf.predict(X_te)
 df=pd.DataFrame(zip(PassengerId,pred),columns=['PassengerId',"Survived"])
 df
df.to_csv('/kaggle/working/output.csv',index=False )<load_from_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
d=pd.read_csv('/kaggle/working/output.csv')
 d<set_options>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
pd.options.display.max_rows=200
pd.set_option('mode.chained_assignment', None)
 simplefilter("ignore", category=ConvergenceWarning)
 simplefilter("ignore", category=RuntimeWarning)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train = pd.read_csv('/kaggle/input/titanic/train.csv', index_col='PassengerId')
 test = pd.read_csv('/kaggle/input/titanic/test.csv', index_col='PassengerId' )<prepare_x_and_y>
n_comp = 29
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 4
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
y_train = train.Survived.copy()
train = train.drop('Survived', axis=1)
 X_test = test<categorify>
remove_vehicle = True
if remove_vehicle:
train_features = train_features.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train_features
def name_labeling(df):
for i in ['Mr.', 'Mrs.', 'Miss', 'Master', 'Army', 'Revered/Important', 'rare', 'Doctor']:
if i == 'Army':
df.Name[df.Name.str.contains(pat='(Major.|Col.|Capt.) ', regex=True)] = 'Army'
elif i == 'Revered/Important':
df.Name[df.Name.str.contains(pat='(Rev.|Countess.|Jonkheer.|Sir.|Lady.) ', regex=True)] = 'Revered/Important'
elif i == 'rare':
df.Name[df.Name.str.contains(pat='(Mme.|Ms.|Mlle.|Don.|Dona.) ', regex=True)] = 'rare'
elif i == 'Doctor':
df.Name[df.Name.str.contains(pat='Dr.')] = 'doctor'
else:
df.Name[df.Name.str.contains(pat=f'{i}', regex=False)] = i
return df<categorify>
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored )
def ticket_labeling(df):
for label, pattern in [('ca', 'C[.]?A[.]?'),('soton', 'SOTON'),('ston', 'STON'),('wc', 'W[.]?[/]?C'), 
('sc', 'S[.]?C[.]?'),('a', 'A[.]?'),('soc', 'S[.]?O[.]?[/]?C'),('pp', 'PP'), 
('fc', '(F.C.|F.C.C.) '),('rest_char', '[A-Z]'),('small_serial_number', '^\d{3,5}$'), 
('large_serial_number', '^\d{6,7}$')]:
df.Ticket[df.Ticket.str.contains(pattern)] = label
return df 
<categorify>
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32 )
def cabin_labeling(df):
for i in ['A', 'B', 'C', 'D', 'E', 'F', 'G']:
df.Cabin[df.Cabin.str.contains(i, na=False)] = i
df.Cabin.fillna('missing', inplace=True)
 return df<create_dataframe>
def evals(model, X, y, verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.clamp(y_preds, 0.0,1.0 ).detach().numpy()
score = log_loss_multi(y, y_preds)
 return y_preds, score
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
temp = cabin_labeling(train.copy() )<groupby>
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
temp = temp.groupby(['Pclass', 'Cabin'])[['Name']].count().rename(columns={'Name':'Passengers'} )<filter>
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
temp = temp.reset_index()
temp_no_missing_value = temp[temp.Cabin != 'missing']
temp_missing_value = temp[temp.Cabin == 'missing']<concatenate>
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
def combined_labeling(df):
return cabin_labeling(ticket_labeling(name_labeling(df)) )<prepare_x_and_y>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': stopping_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
X_train = combined_labeling(train.copy())
 X_test = combined_labeling(test.copy() )<groupby>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
def proportions(df):

 df = df.groupby(['Pclass', 'Cabin'])['Name'].count().reset_index().rename(columns={'Name':'Passengers'})
 total_passengers_in_cabins = df.Passengers[df.Cabin != 'missing'].sum()
cabin_proportions = df['Passengers'][df.Cabin != 'missing'] / total_passengers_in_cabins
return cabin_proportions
def no_cabins(df):

 df = df.groupby(['Pclass', 'Cabin'])['Name'].count().reset_index().rename(columns={'Name':'Passengers'})
 return df.Passengers[df.Cabin == 'missing']
def cabin_imputer(df, missing_vals, cabin_proportions):
if all(df.Pclass == 1):
imputation_ndarray = np.random.choice(['A', 'B', 'C', 'D', 'E', 'T'], size=missing_vals[0], p=cabin_proportions[1])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
elif all(df.Pclass == 2):
imputation_ndarray = np.random.choice(['D', 'E', 'F'], size=missing_vals[1], p=cabin_proportions[2])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
elif all(df.Pclass == 3):
imputation_ndarray = np.random.choice(['E', 'F', 'G'], size=missing_vals[2], p=cabin_proportions[3])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
def imputer(df):
df.Age.fillna(df.groupby(['Pclass', 'Sex'] ).Age.transform('mean'), inplace=True)
 cabin_proportions = X_train.groupby(['Pclass'] ).apply(proportions)
 missing_vals = list(df.groupby(['Pclass'] ).apply(no_cabins))
missing_val_imputed_series = df.groupby(['Pclass'] ).apply(partial(cabin_imputer,
missing_vals=missing_vals,
cabin_proportions=cabin_proportions))
missing_val_imputed_series = missing_val_imputed_series.reset_index().set_index('PassengerId')
 missing_val_imputed_series = missing_val_imputed_series.drop('Pclass', axis=1 ).sort_index().iloc[:, 0]
for index, val in zip(missing_val_imputed_series.index, missing_val_imputed_series):
df.Cabin[index] = val
for feature in df.columns:
df[feature].fillna(df[feature].mode() [0], inplace=True)
 df.Cabin[df.Cabin == 'T'] = 'E'
df.reset_index(drop=True, inplace=True)
 return df<categorify>
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()
X_train_imputed = imputer(X_train.copy())
 X_test_imputed = imputer(X_test.copy() )<normalization>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 10
self.EPOCHS = 200
self.num_ensembling = 1
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = ".. /output/kaggle/working/model"
self.strategy = "KFOLD"
cfg = Config()
class FeatureEngineering(BaseEstimator, TransformerMixin):
def __init__(self, drop_Cabin=False, drop_Name=False, Embarked_target=False, SibSp_Parch_simplify=True,
drop_Ticket=False, scaler='MinMaxScaler', smoothing=10, test=False):
self.drop_Cabin = drop_Cabin
self.drop_Name = drop_Name
self.drop_Ticket = drop_Ticket
self.Embarked_target = Embarked_target
self.SibSp_Parch_simplify = SibSp_Parch_simplify
self.scaler = scaler
self.smoothing = smoothing
self.target_encoder = None
self.test = test
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Pclass'] = X.Pclass.astype(str)
 dummies = pd.get_dummies(X.loc[:, ['Pclass', 'Sex']])
 X = pd.concat([X, dummies], axis=1 ).drop(['Pclass', 'Sex'], axis=1)
 if not self.test and len(X)> 400:
self.cols = ['Ticket', 'Name', 'Cabin'] if not self.Embarked_target else ['Cabin', 'Name', 'Ticket', 'Embarked']
self.target_encoder = TargetEncoder(cols=self.cols, smoothing=self.smoothing)
 self.target_encoder.fit(X.loc[:, self.cols], y_train.iloc[X.index])
 X.loc[:, self.cols] = self.target_encoder.transform(X.loc[:, self.cols])
 else:
X.loc[:, self.cols] = self.target_encoder.transform(X.loc[:, self.cols])
 if self.drop_Cabin:
X.drop('Cabin', axis=1, inplace=True)
 if self.drop_Name:
X.drop('Name', axis=1, inplace=True)
 if self.drop_Ticket:
X.drop('Ticket', axis=1, inplace=True)
 if self.SibSp_Parch_simplify:
X['SibSp'].replace({0:'No', 1:'Yes', 2:'Yes', 3:'Yes', 4:'Yes',
5:'Yes', 6:'Yes', 7:'Yes', 8:'Yes', 9:'Yes'}, inplace=True)
 X['Parch'].replace({0:'No', 1:'Yes', 2:'Yes', 3:'Yes', 4:'Yes',
5:'Yes', 6:'Yes', 7:'Yes', 8:'Yes', 9:'Yes'}, inplace=True)
 dummies = pd.get_dummies(X.loc[:, ['SibSp', 'Parch']])
 X = pd.concat([X, dummies], axis=1 ).drop(['SibSp', 'Parch'], axis=1)
 X = pd.get_dummies(X)
 features_to_scale = [feature for feature in X.columns if X[feature].max() != 1 and X[feature].min() != 1]
if not self.test and len(X)> 400:
if self.scaler == 'StandardScaler':
self.scale_transformer = StandardScaler()
elif self.scaler == 'MinMaxScaler':
self.scale_transformer = MinMaxScaler()
elif self.scaler == 'RobustScaler':
self.scale_transformer = RobustScaler()
scaled_features = self.scale_transformer.fit_transform(X.loc[:, features_to_scale])
 X.loc[:, features_to_scale] = pd.DataFrame(data=scaled_features, columns=features_to_scale, index=X.index)
 else:
scaled_features = self.scale_transformer.fit_transform(X.loc[:, features_to_scale])
 X.loc[:, features_to_scale] = pd.DataFrame(data=scaled_features, columns=features_to_scale, index=X.index)
 return X 
<create_dataframe>
X_test = np.concatenate([cat_test, numerical_test ], axis=1 )
def results(cv_results_, n):
df = pd.DataFrame(cv_results_)[['params', 'mean_test_score']].nlargest(n, columns='mean_test_score')
 for i in range(len(df)) :
print(f'{df.iloc[i, 0]} : {df.iloc[i, 1]}' )<define_search_model>
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
for seed in range(cfg.num_ensembling):
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, lambda_sparse=0,
cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax', device_name=cfg.device,
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val,max_epochs=cfg.EPOCHS, patience=50,
batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=LabelSmoothing(0.001)
)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
param_grid = {'feature_engineering__drop_Cabin':[True, False],
'feature_engineering__drop_Ticket':[True, False],
'feature_engineering__drop_Name':[True, False],
'feature_engineering__Embarked_target':[True, False],
'feature_engineering__SibSp_Parch_simplify':[False],
'feature_engineering__scaler':['StandardScaler'],
'feature_engineering__smoothing':[5]}
feature_engineering_pipeline = Pipeline([('feature_engineering', FeatureEngineering()), 
('model', LogisticRegression())])
 grid = GridSearchCV(feature_engineering_pipeline, param_grid,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
grid.fit(X_train_imputed.copy() , y_train )<choose_model_class>
if cfg.strategy != "KFOLD":
i = 0
mskf = MultilabelStratifiedShuffleSplit(n_splits=1000, test_size=0.1, random_state=0)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
if i == cfg.SPLITS:
break
if not check_targets(targets_tr[train_idx]):
continue
print("FOLDS : ", i, j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_dims=cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=1e-3, amsgrad=True), mask_type="sparsemax", device_name=cfg.device)
 name = cfg.save_name + f"_{j}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 i+=1
p = np.stack(p)
 preds_test.append(p)
 preds_test = np.stack(preds_test)
fe = FeatureEngineering(drop_Cabin=True, drop_Name=False, drop_Ticket=False, Embarked_target=False, SibSp_Parch_simplify=False,
scaler='StandardScaler', smoothing=5 )<categorify>
if cfg.strategy != "KFOLD":
print("auc mean : ", sum(scores_auc)/len(scores_auc))
print("CV score : ", log_loss_multi(np.concatenate(oof_targets), np.concatenate(oof_preds)) )
X_train_fe = fe.fit_transform(X_train_imputed.copy() )<feature_engineering>
print(model )
fe.test = True<categorify>
submission[columns] = preds_test.mean(1 ).mean(0)
 submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
X_test_fe = fe.transform(X_test_imputed.copy() )<train_on_grid>
submission.to_csv("submission.csv", index=False )
def parameter_plot(model, X, y, n_estimators=[100, 200, 300, 400, 600, 900, 1300, 1700, 2000, 2500], hyper_param=None, **kwargs):
param_name, param_vals = hyper_param
param_grid = {'n_estimators':n_estimators,
f'{param_name}':param_vals}
grid = GridSearchCV(model(**kwargs), param_grid,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', n_jobs=-1, verbose=2)
 grid.fit(X, y)
 results = pd.DataFrame(grid.cv_results_)['mean_test_score'].values
results = results.reshape(len(param_vals), len(n_estimators))
plt.figure(figsize=(15, 9))
for i in range(1, len(param_vals)+ 1):
plt.plot(n_estimators, results[i-1], label=f'{param_name} - {param_vals[i-1]}')
 plt.legend()
plt.show()<split>
submission.to_csv("submission.csv", index=False )
def learning_curve_plotter(Model, X, y, params_1, params_2, step=50):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
 plt.figure(figsize=(16, 7))
for i,(name, params)in enumerate([params_1, params_2]):
train_score = []
val_score = []
plt.subplot(1, 2, i+1)
 for j in range(100, len(X_train), step):
model = Model(**params ).fit(X_train[:j], y_train[:j])
 train_score.append(model.score(X_train[:j], y_train[:j]))
val_score.append(model.score(X_test, y_test))
plt.plot(train_score, 'r-', label='Training accuracy')
 plt.plot(val_score, 'b-', label='Validation accuracy')
 plt.title(f'{name}')
 plt.xlabel('Training set size')
 plt.ylabel('Accuracy')
 plt.legend()
plt.show()<init_hyperparams>
model.reducing_matrix