Kaggle competition: Mechanisms of Action (MoA) Prediction
<compute_test_metric>
warnings.filterwarnings('ignore' )
def calculateClassificationScores(y_true, y_predicted, model, X_test, average='macro'):
accuracy = accuracy_score(y_true, y_predicted)
 f1 = f1_score(y_true, y_predicted, average=average)
 precision = precision_score(y_true, y_predicted, average=average)
 recall = recall_score(y_true, y_predicted, average=average)
 if model and isinstance(model, CatBoostClassifier):
score = model.get_best_score()
if score.get('validation_0'):
multiclass = score['validation_0']['MultiClass']
else:
multiclass = model.score(X_test, y_true)
 return(accuracy, f1, precision, recall, multiclass)
 return(accuracy, f1, precision, recall)
 def crossValidation(params, X, y):
f1_scores = []
accuracy_scores = []
precision_scores = []
recall_scores = []
multiclass_scores = []
for train_index, val_index in skf.split(X.values, y.values):
X_train = X[X.index.isin(train_index)]
X_train = FeaturesData(
 num_feature_data=X_train[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_train[categoricalFeatures].__array__(dtype=object)
)
 y_train = y[y.index.isin(train_index)].astype('int' ).values
X_valid = X[X.index.isin(val_index)]
X_valid = FeaturesData(
 num_feature_data=X_valid[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_valid[categoricalFeatures].__array__(dtype=object)
)
 y_valid = y[y.index.isin(val_index)].astype('int' ).values
pool_test = Pool(X_valid, label=y_valid)
 clf = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=params['depth'],
iterations=params['iterations'],
l2_leaf_reg=params['l2_leaf_reg'],
learning_rate=params['learning_rate'],
task_type='CPU' 
)
 clf.fit(X_train, y_train, eval_set=pool_test, use_best_model=True)
 y_pred = clf.predict(X_valid)
(accuracy, f1, precision, recall, multiclass)= calculateClassificationScores(y_valid, y_pred, clf, X_valid)
 multiclass_scores.append(multiclass)
 accuracy_scores.append(accuracy)
 f1_scores.append(f1)
 precision_scores.append(precision)
 recall_scores.append(recall)
 return(multiclass_scores, accuracy_scores, f1_scores, precision_scores, recall_scores)
 def searchBestParams(grid, X, y):
catboostDf = pd.DataFrame({
'model':[],
'multiclass_score_mean':[],
'multiclass_score_std':[],
'f1_score_mean':[],
'f1_score_std':[],
'accuracy_score_mean':[],
'accuracy_score_std':[],
'precision_score_mean':[],
'precision_score_std':[],
'recall_score_mean':[],
'recall_score_std':[],
'params': []} 
)
 for params in grid:
print(params)
(multiclass_scores, accuracy_scores, f1_scores, precision_scores, recall_scores)= crossValidation(params, X, y)
 catboostDf = catboostDf.append({
'multiclass_score_mean': np.mean(multiclass_scores),
'multiclass_score_std': np.std(multiclass_scores),
'f1_score_mean': np.mean(f1_scores),
'f1_score_std': np.std(f1_scores),
'accuracy_score_mean': np.mean(accuracy_scores),
'accuracy_score_std': np.std(accuracy_scores),
'precision_score_mean': np.mean(precision_scores),
'precision_score_std': np.std(precision_scores),
'recall_score_mean': np.mean(recall_scores),
'recall_score_std': np.std(recall_scores),
'params': params
}, ignore_index=True)
 return catboostDf<define_variables>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
<split>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
TARGETS = train_targets_scored.columns[1:]
def trainCatboost(numericalFeatures, categoricalFeatures, target, data, params=None):
print(numericalFeatures)
 print(categoricalFeatures)
 print(target)
 print(data.size)
 X_train, X_test, y_train, y_test = train_test_split(
 data[numericalFeatures + categoricalFeatures],
data[target],
test_size=0.25,
random_state=42 
)
 X_train_pool = FeaturesData(
 num_feature_data=X_train[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_train[categoricalFeatures].__array__(dtype=object)
)
 y_train_pool = y_train.astype('int' ).values
X_test_pool = FeaturesData(
 num_feature_data=X_test[numericalFeatures].astype('float32' ).values,
cat_feature_data=X_test[categoricalFeatures].__array__(dtype=object)
)
 y_test_pool = y_test.astype('int' ).values
model = None
if params:
model = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=params['depth'],
iterations=params['iterations'],
l2_leaf_reg=params['l2_leaf_reg'],
learning_rate=params['learning_rate'],
task_type='GPU',
class_weights=[4, 1, 1, 1, 1] 
)
 else:
model = CatBoostClassifier(
 loss_function='MultiClass',
verbose=False,
depth=8,
iterations=140,
l2_leaf_reg=12,
learning_rate=0.15,
task_type='GPU',
class_weights=[4, 1, 1, 1, 1] 
)
 model.fit(X_train_pool, y_train_pool,logging_level='Silent')
 y_predicted = model.predict(X_test_pool)
 return(model, y_predicted, X_train, y_train, X_train_pool, X_test_pool, y_test )<compute_test_metric>
def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=1903 )
<define_variables>
TARGETS_SUB = [TARGETS]
<compute_test_metric>
positive_threshold = 20
count_positive_by_feature = train_targets_scored.sum(axis=0)[1:]
low_positive_by_feature = list(count_positive_by_feature[count_positive_by_feature <= positive_threshold].index)
 for feature in low_positive_by_feature:
train_targets_scored_sub = train_targets_scored[train_targets_scored[feature] == 1]
train_features_sub = train_features.merge(train_targets_scored_sub, on='sig_id')
 train_features_sub_list = []
copy_time = int(positive_threshold / len(train_targets_scored_sub)) + 1
for ij in range(copy_time):
train_features_sub_copy = train_features_sub.copy()
train_features_sub_copy['key'] = train_features_sub_copy['sig_id'] + str(ij)
 train_features_sub_list.append(train_features_sub_copy)
 train_features_sub_fake = pd.concat(train_features_sub_list)
 train_features_sub_fake['sig_id'] = train_features_sub_fake['key']
train_features_sub_fake = train_features_sub_fake.drop(['key'], axis=1)
 train_features = pd.concat([train_features, train_features_sub_fake[train_features.columns]])
 train_targets_scored = pd.concat([train_targets_scored, train_features_sub_fake[train_targets_scored.columns]])
 train_features = train_features.reset_index(drop=True)
 train_targets_scored = train_targets_scored.reset_index(drop=True )
<train_on_grid>
for col in TARGETS:
train_targets_scored[col][train_targets_scored[col] <= 0.0001] = 0.0001
train_targets_scored[col][train_targets_scored[col] >= 0.9999] = 0.9999
<feature_engineering>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
<data_type_conversions>
n_comp = 463
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
PCA_GENES = [f'pca_G-{i}' for i in range(n_comp)]
train2 = pd.DataFrame(train2, columns=PCA_GENES)
 test2 = pd.DataFrame(test2, columns=PCA_GENES)
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 60
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
PCA_CELLS = [f'pca_C-{i}' for i in range(n_comp)]
train2 = pd.DataFrame(train2, columns=PCA_CELLS)
 test2 = pd.DataFrame(test2, columns=PCA_CELLS)
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
<save_to_csv>
var_thresh = VarianceThreshold(threshold=0.9)
 data = train_features.append(test_features)
 data_gene = data.copy()
data_cell = data.copy()
data_gene = data_gene[GENES+PCA_GENES]
data_cell = data_cell[CELLS+PCA_CELLS]
data_transformed_gene = var_thresh.fit_transform(data_gene)
 train_features_transformed_gene = data_transformed_gene[ : train_features.shape[0]]
test_features_transformed_gene = data_transformed_gene[-test_features.shape[0] : ]
train_features_transformed_gene = pd.DataFrame(train_features_transformed_gene)
 test_features_transformed_gene = pd.DataFrame(test_features_transformed_gene)
 train_features_transformed_gene.rename(columns=lambda x: 'vargen_' + str(x), inplace=True)
 test_features_transformed_gene.rename(columns=lambda x: 'vargen_' + str(x), inplace=True)
 data_transformed_cell = var_thresh.fit_transform(data_cell)
 train_features_transformed_cell = data_transformed_cell[ : train_features.shape[0]]
test_features_transformed_cell = data_transformed_cell[-test_features.shape[0] : ]
train_features_transformed_cell = pd.DataFrame(train_features_transformed_cell)
 test_features_transformed_cell = pd.DataFrame(test_features_transformed_cell)
 train_features_transformed_cell.rename(columns=lambda x: 'varcell_' + str(x), inplace=True)
 test_features_transformed_cell.rename(columns=lambda x: 'varcell_' + str(x), inplace=True)
 train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, train_features_transformed_gene, train_features_transformed_cell], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, test_features_transformed_gene, test_features_transformed_cell], axis=1)
 train_features
def generateSubmissionCatboost(model, numericalFeatures, categoricalFeatures, test, fileName):
X = test[numericalFeatures + categoricalFeatures]
X = FeaturesData(
 num_feature_data=X[numericalFeatures].astype('float32' ).values,
cat_feature_data=X[categoricalFeatures].__array__(dtype=object)
)
 predictions = model.predict(X)
 test['AdoptionSpeed'] = predictions
test.AdoptionSpeed = test['AdoptionSpeed'].map({0.0: '0', 1.0: '1', 2.0: '2', 3.0: '3', 4.0: '4'})
 test[['PetID', 'AdoptionSpeed']].to_csv(fileName + '.csv', index=False )<compute_test_metric>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
<init_hyperparams>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )
<compute_test_metric>
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
}
return dct
<drop_column>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs_gen, inputs_cell, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(inputs_gen, inputs_cell)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs_gen, inputs_cell, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(inputs_gen, inputs_cell)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs_gen, inputs_cell = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(inputs_gen, inputs_cell)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
to_drop_columns = ['PetID', 'Name', 'RescuerID']
X_train_non_null = X_train_non_null.drop(to_drop_columns, axis=1 )<drop_column>
dropout_ratio = 0.25
class Model(nn.Module):
def __init__(self, num_features, num_gen_features, num_cell_features, num_targets, hidden_size):
super(Model, self ).__init__()

 self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(dropout_ratio)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(dropout_ratio)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(dropout_ratio)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.cell_dropout2 = nn.Dropout(dropout_ratio)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size * 2)
 self.dropout3 = nn.Dropout(dropout_ratio)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size * 2, num_targets))
def forward(self, gen_x, cell_x):

 gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.gen_dense2(cell_x))
x = torch.cat(( gen_x, cell_x), dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
testIds = X_test_non_null['PetID']
X_test_non_null = X_test_non_null.drop(to_drop_columns, axis=1 )<drop_column>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
X_test_non_null = X_test_non_null.drop(['AdoptionSpeed'], axis=1 )<import_modules>
feature_cols = [c for c in process_data(folds ).columns if c not in TARGETS]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
common_cols = ['cp_time_24', 'cp_time_48', 'cp_time_72', 'cp_dose_D1', 'cp_dose_D2']
gen_features = [c for c in feature_cols if(c.startswith('vargen_')or(c in common_cols)) ]
cell_features = [c for c in feature_cols if(c.startswith('varcell_')or(c in common_cols)) ]
print(len(feature_cols), len(gen_features), len(cell_features))
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=2048
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
def run_training(fold, seed, target_cols):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train_gen, x_train_cell, y_train = train_df[gen_features].values, train_df[cell_features].values, train_df[target_cols].values
x_valid_gen, x_valid_cell, y_valid = valid_df[gen_features].values, valid_df[cell_features].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train_gen, x_train_cell, y_train)
 valid_dataset = MoADataset(x_valid_gen, x_valid_cell, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_gen_features=len(gen_features),
num_cell_features=len(cell_features),
num_targets=len(target_cols),
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, eps=0.00001)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), len(target_cols)))
 best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test_gen, x_test_cell = test_[gen_features].values, test_[cell_features].values
testdataset = TestDataset(x_test_gen, x_test_cell)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_gen_features=len(gen_features),
num_cell_features=len(cell_features),
num_targets=len(target_cols),
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<prepare_x_and_y>
def run_k_fold(NFOLDS, seed, target_cols):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed, target_cols)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
for target_cols in TARGETS_SUB:
SEED = [1903, 1881]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed, target_cols)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 predictions[predictions<=0.0001] = 0.0001
predictions[predictions>=0.9999] = 0.9999
train[target_cols] = oof
test[target_cols] = predictions
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<train_on_grid>
train_targets_scored_orig = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 valid_results = train_targets_scored_orig.drop(columns=list(TARGETS)).merge(train[['sig_id']+list(TARGETS)], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored_orig[list(TARGETS)].values
y_pred = valid_results[list(TARGETS)].values
score = 0
for i in range(len(TARGETS)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
optR = OptimizedRounder()
optR.fit(oof_train, X_train_non_null['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train_non_null['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
sub = sample_submission.drop(columns=list(TARGETS)).merge(test[['sig_id']+list(TARGETS)], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )