Kaggle competition: Mechanisms of Action (MoA) Prediction
pred_xgb_pseudo = pd.DataFrame()
for tr_id, vl_id in kf2.split(X_xgb, y_xgb):
print("========================================================================")
 X_train, X_val = X_xgb[tr_id, :], X_xgb[vl_id, :]
y_train, y_val = y_xgb[tr_id], y_xgb[vl_id]
d_train = xgb.DMatrix(X_train, y_train)
 d_val = xgb.DMatrix(X_val, y_val)
 model = xgb.train(params=params_xgb,
dtrain=d_train,
num_boost_round=10000,
early_stopping_rounds=50,
verbose_eval=500,
evals=[(d_train, "train"),(d_val, "val")])
 pred = model.predict(d_test, ntree_limit=model.best_ntree_limit)
 pred = pd.Series(pred)
 pred_xgb_pseudo = pd.concat([pred_xgb_pseudo, pred], axis=1 )<init_hyperparams>
NB = '101'
IS_TRAIN = False
MODEL_DIR = ".. /input/503-203-tabnet-with-nonscored-features-train/model"
INT_DIR = "interim"
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
params_lgb = {
"task": "train",
"boosting_type": "gbdt",
"objective": "regression",
"metric": "rmse",
"learning_rate": 0.0057,
"num_leaves": 256,
"bagging_fraction": 0.8206341150202605,
"feature_fraction": 0.5,
"min_data_in_leaf": 100,
"lambda_l1": 1.074622455507616e-05,
"lambda_l2": 2.0521330798729704e-06,
"min_data_per_group": 5,
"max_depth": -1,
"subsample_for_bin": 200000,
"cat_smooth": 1.0,
"min_sum_hessian_in_leaf": 0.001,
"bagging_freq": 6,
"min_gain_to_split": 0.0,
"verbosity": -1,
"random_state": SEED
}<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
y_test_lgb = pd.DataFrame()
for tr_id, vl_id in kf1.split(X, y):
print("======================================================================")
 X_train, X_val = X[tr_id, :], X[vl_id, :]
y_train, y_val = y[tr_id], y[vl_id]
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val, reference=lgb_train)
 model = lgb.train(params=params_lgb,
train_set=lgb_train,
valid_sets=(lgb_train, lgb_val),
num_boost_round=10000,
early_stopping_rounds=50,
verbose_eval=500)
 pred = model.predict(X_test, num_iteration=model.best_iteration)
 pred = pd.Series(pred)
 y_test_lgb = pd.concat([y_test_lgb, pred], axis=1 )<prepare_output>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pred_lgb = y_test_lgb<compute_test_metric>
n_comp = 50
n_dim = 15
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[GENES])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_g.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_g.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_g.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_g.pkl")
 data2 = pca.transform(data[GENES])
 data3 = umap.transform(data[GENES])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_G-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
 n_comp = 15
n_dim = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 umap = UMAP(n_components=n_dim, random_state=DEFAULT_SEED ).fit(train_features[CELLS])
 pd.to_pickle(pca, f"{MODEL_DIR}/{NB}_pca_c.pkl")
 pd.to_pickle(umap, f"{MODEL_DIR}/{NB}_umap_c.pkl")
 else:
pca = pd.read_pickle(f"{MODEL_DIR}/{NB}_pca_c.pkl")
 umap = pd.read_pickle(f"{MODEL_DIR}/{NB}_umap_c.pkl")
 data2 = pca.transform(data[CELLS])
 data3 = umap.transform(data[CELLS])
 train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train3 = data3[:train_features.shape[0]]
test3 = data3[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train3 = pd.DataFrame(train3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test3 = pd.DataFrame(test3, columns=[f'umap_C-{i}' for i in range(n_dim)])
 train_features = pd.concat(( train_features, train2, train3), axis=1)
 test_features = pd.concat(( test_features, test2, test3), axis=1)
y_test_lgb = y_test_lgb.mean(axis=1)
 y_test_lgb = np.array(y_test_lgb )<train_model>
if IS_TRAIN:
var_thresh = VarianceThreshold(threshold=0.5 ).fit(train_features.iloc[:, 4:])
 pd.to_pickle(var_thresh, f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 else:
var_thresh = pd.read_pickle(f"{MODEL_DIR}/{NB}_variance_thresh0_5.pkl")
 data = train_features.append(test_features)
 data_transformed = var_thresh.transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print(train_features.shape)
 print(test_features.shape )
pred_lgb_pseudo = pd.DataFrame()
for tr_id, vl_id in kf2.split(X_lgb, y_lgb):
print("======================================================================")
 X_train, X_val = X_lgb[tr_id, :], X_lgb[vl_id, :]
y_train, y_val = y_lgb[tr_id], y_lgb[vl_id]
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val, reference=lgb_train)
 model = lgb.train(params=params_lgb,
train_set=lgb_train,
valid_sets=(lgb_train, lgb_val),
num_boost_round=10000,
early_stopping_rounds=50,
verbose_eval=500)
 pred = model.predict(X_test, num_iteration=model.best_iteration)
 pred = pd.Series(pred)
 pred_lgb_pseudo = pd.concat([pred_lgb_pseudo, pred], axis=1 )<concatenate>
train = train_features[train_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
pred = pd.concat([pred_xgb, pred_lgb], axis=1)
 pred = pred.mean(axis=1)
 pred_pseudo = pd.concat([pred_xgb_pseudo, pred_lgb_pseudo], axis=1)
 pred_pseudo = pred_pseudo.mean(axis=1 )<load_from_csv>
train.to_pickle(f"{INT_DIR}/{NB}_train_preprocessed.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_preprocessed.pkl" )
sample_sub = pd.read_csv(".. /input/tabular-playground-series-jan-2021/sample_submission.csv")
 sub = sample_sub.copy()
sub_pseudo = sample_sub.copy()<prepare_output>
NB = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
HIDDEN_SIZE = 2048
EPOCHS = 15
BATCH_SIZE = 2048
NFOLDS = 10
NREPEATS = 1
NSEEDS = 5
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2
sub["target"] = pred
sub<prepare_output>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def calc_valid_log_loss(train, target, target_cols):
y_pred = train[target_cols].values
y_true = target[target_cols].values
y_true_t = torch.from_numpy(y_true.astype(np.float64)).clone()
y_pred_t = torch.from_numpy(y_pred.astype(np.float64)).clone()
return torch.nn.BCELoss()(y_pred_t, y_true_t ).to('cpu' ).detach().numpy().copy()
sub_pseudo["target"] = pred_pseudo
sub_pseudo<save_to_csv>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=HIDDEN_SIZE):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
sub.to_csv("submission.csv", index=False)
 sub_pseudo.to_csv("submission_pseudo.csv", index=False )<set_options>
def run_training(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=PCT_START, div_factor=DIV_FACS,
max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
best_loss_epoch = -1
if IS_TRAIN:
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if valid_loss < best_loss:
best_loss = valid_loss
best_loss_epoch = epoch
oof[val_idx] = valid_preds
model.to('cpu')
 torch.save(model.state_dict() , f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth")
 model.to(DEVICE)
 if epoch % 10 == 0 or epoch == EPOCHS-1:
print(f"seed: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}, best_loss: {best_loss:.6f}, best_loss_epoch: {best_loss_epoch}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.load_state_dict(torch.load(f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth"))
model.to(DEVICE)
 if not IS_TRAIN:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
%matplotlib inline
sns.set_style('whitegrid')
 plt.style.use('seaborn-deep')
 plt.rcParams['font.family'] = 'sans-serif'
plt.rcParams['font.serif'] = 'Ubuntu'
plt.rcParams['font.monospace'] = 'Ubuntu Mono'
plt.rcParams['font.size'] = 10
plt.rcParams['axes.labelsize'] = 12
plt.rcParams['axes.titlesize'] = 12
plt.rcParams['xtick.labelsize'] = 8
plt.rcParams['ytick.labelsize'] = 8
plt.rcParams['legend.fontsize'] = 12
plt.rcParams['figure.titlesize'] = 14
plt.rcParams['figure.figsize'] =(12, 8)
 pd.options.mode.chained_assignment = None
pd.options.display.float_format = '{:.2f}'.format
pd.set_option('display.max_columns', 200)
 pd.set_option('display.width', 400)
 warnings.filterwarnings('ignore')
 seed = 12
np.random.seed(seed)
<load_from_csv>
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = RepeatedMultilabelStratifiedKFold(n_splits=NFOLDS, n_repeats=NREPEATS, random_state=None)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
base = '/kaggle/input/tabular-playground-series-jan-2021/'
data_file = base + "train.csv"
df = pd.read_csv(data_file)
 df.head()<load_from_csv>
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions
data_file = base + "test.csv"
df_test = pd.read_csv(data_file)
 df_test.head()<drop_column>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
df.drop('id', inplace=True, axis=1 )<remove_duplicates>
train = pd.read_pickle(f"{INT_DIR}/101_train_preprocessed.pkl")
 test = pd.read_pickle(f"{INT_DIR}/101_test_preprocessed.pkl" )
print(df.shape)
 df.drop_duplicates(inplace=True)
 print(df.shape )<count_missing_values>
train_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_train_preprocessed.pkl")
 test_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_test_preprocessed.pkl" )
df.isna().any().sum()<sort_values>
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
print(train_targets_nonscored.shape)
 train = train.merge(train_targets_nonscored, on='sig_id' )
skewed_features = df.apply(lambda x: x.skew() ).sort_values(ascending=False)
 skewed_features<feature_engineering>
run_seeds(train, test, feature_cols, target_cols )
<split>
valid_loss_total = calc_valid_log_loss(train, target, target_cols)
 print(f"CV loss: {valid_loss_total}" )
df_shuffle = df.sample(frac=1, random_state=seed ).reset_index(drop=True)
 df_y = df_shuffle.pop('target')
 df_X = df_shuffle
X_train, X_test, y_train, y_test = skms.train_test_split(df_X, df_y, train_size=0.9, random_state=seed)
 print(f"Train set has {X_train.shape[0]} records out of {len(df_shuffle)} which is {round(X_train.shape[0]/len(df_shuffle)*100)}%")
 print(f"Test set has {X_test.shape[0]} records out of {len(df_shuffle)} which is {round(X_test.shape[0]/len(df_shuffle)*100)}%" )<import_modules>
train.to_pickle(f"{INT_DIR}/{NB}_train_nonscored_pred.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_nonscored_pred.pkl" )
import sklearn.linear_model as sklm<normalization>
valid_results = train_targets_nonscored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_nonscored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
scaler = skp.MinMaxScaler()
X_train = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns)
 X_test = pd.DataFrame(scaler.transform(X_test), columns=X_train.columns)
 X_train.describe()<compute_test_metric>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
def expm1(x):
return np.expm1(x)
 def getRmse(y_train, y_train_pred):
print(skm.mean_squared_error(y_train, y_train_pred))<compute_train_metric>
from pytorch_tabnet.tab_model import TabNetRegressor
lmr = sklm.Ridge(alpha=0.001)
 lmr.fit(X_train, y_train)
 y_train_pred = lmr.predict(X_train)
 y_test_pred = lmr.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<train_on_grid>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
params = {'alpha': [0.0001, 0.001, 0.005, 0.01, 0.03, 0.05, 0.1, 0.5, 1.0, 5.0, 10]}
ridge = sklm.Ridge()
model_cv_ridge = skms.GridSearchCV(estimator = ridge, n_jobs=-1, param_grid = params,
scoring= 'neg_mean_squared_error', cv = 5,
return_train_score=True, verbose = 3)
 model_cv_ridge.fit(X_train, y_train)
 print(model_cv_ridge.best_estimator_)
 y_train_pred = model_cv_ridge.predict(X_train)
 y_test_pred = model_cv_ridge.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<train_model>
NB = '503'
NB_PREV = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
NFOLDS = 10
NREPEATS = 1
NSEEDS = 3
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2
cbr = cb.CatBoostRegressor(loss_function='RMSE', verbose=0)
 cbr.fit(X_train, y_train, eval_set=(X_test, y_test))
print(cbr.best_score_)
 y_train_pred = cbr.predict(X_train)
 y_test_pred = cbr.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<predict_on_test>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
xgb = ske.GradientBoostingRegressor(criterion='mse', random_state=1)
 xgb.fit(X_train, y_train)
 y_train_pred = xgb.predict(X_train)
 y_test_pred = xgb.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<predict_on_test>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
xgb = ske.ExtraTreesRegressor(criterion='mse', random_state=1)
 xgb.fit(X_train, y_train)
 y_train_pred = xgb.predict(X_train)
 y_test_pred = xgb.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<predict_on_test>
use_test_for_preprocessing = False
for col in(GENES + CELLS):
if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 if use_test_for_preprocessing:
raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 transformer.fit(raw_vec)
 else:
raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
xgb = ske.RandomForestRegressor(criterion='mse', random_state=1)
 xgb.fit(X_train, y_train)
 y_train_pred = xgb.predict(X_train)
 y_test_pred = xgb.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<predict_on_test>
n_comp = 90
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[GENES])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 data2 =(fa.transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[CELLS])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 data2 =(fa.transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
xgb = xg.XGBRegressor(objective ='reg:squarederror', random_state=1)
 xgb.fit(X_train, y_train)
 y_train_pred = xgb.predict(X_train)
 y_test_pred = xgb.predict(X_test)
 getRmse(y_train, y_train_pred)
 getRmse(y_test, y_test_pred )<set_options>
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 if IS_TRAIN:
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 pd.to_pickle(kmeans, f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 else:
kmeans = pd.read_pickle(f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 train[f'clusters_{kind}'] = kmeans.predict(train_)
 test[f'clusters_{kind}'] = kmeans.predict(test_)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )
print("TF version:-", tf.__version__)
 tf.random.set_seed(seed )<train_model>
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )
epochs = 40
model_1 = k.models.Sequential([
k.layers.Dense(512, activation='relu', input_shape=(X_train.shape[1],)) ,
k.layers.Dropout(0.2),
k.layers.Dense(256, activation='relu'),
k.layers.Dropout(0.2),
k.layers.Dense(1, activation='linear'),
])
 print(model_1.summary())
 model_1.compile(optimizer='adam',
loss='mse',
metrics='mse' 
)
 history = model_1.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs,
callbacks=[callbacks_list] )<prepare_x_and_y>
remove_vehicle = True
if remove_vehicle:
trt_idx = train_features['cp_type']=='trt_cp'
train_features = train_features.loc[trt_idx].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[trt_idx].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[trt_idx].reset_index(drop=True)
 else:
pass
def getTestResults(m=None):
df_final = df.sample(frac=1, random_state=1 ).reset_index(drop=True)
 test_cols = [x for x in df.columns if 'target' not in x]
df_final_test = df_test[test_cols]
df_y = df_final.pop('target')
 df_X = df_final
scaler = skp.MinMaxScaler()
df_X = pd.DataFrame(scaler.fit_transform(df_X), columns=df_X.columns)
 X_test = pd.DataFrame(scaler.transform(df_final_test), columns=df_X.columns)
 if m is None:
lmr = cb.CatBoostRegressor(loss_function='RMSE', verbose=0)
 lmr.fit(df_X, df_y)
 else:
lmr = m
y_train_pred = lmr.predict(df_X)
 y_test_pred = lmr.predict(X_test)
 if m is not None:
y_test_pred = [y[0] for y in y_test_pred]
getRmse(df_y, y_train_pred)
 return y_test_pred
results = getTestResults() 
<prepare_output>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
submission = pd.DataFrame({
'id': df_test['id'],
'target': results,
})
 submission.head()<save_to_csv>
train_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_train_nonscored_pred.pkl')
 test_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_test_nonscored_pred.pkl' )
submission.to_csv('./submission_Catboost.csv', index=False )<set_options>
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
%matplotlib inline<install_modules>
train = train.merge(train_nonscored_pred[train_targets_nonscored.columns], on='sig_id')
 test = test.merge(test_nonscored_pred[train_targets_nonscored.columns], on='sig_id' )
!pip install optuna<load_from_csv>
nonscored_target = [c for c in train_targets_nonscored.columns if c != "sig_id"]
for col in(nonscored_target):
vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<feature_engineering>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
len(feature_cols )
features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
datax = pd.concat([train[features],test[features]], axis=0)
 datax1 = datax.abs()
def featx(data, features):
cx=int(1000000)
 for f in features:
data[f] = data[f]*cx
dmm = data[f].max() -data[f].min()
data[f] =(data[f]-data[f].min())/dmm

 return data
data = featx(datax, features)
 data.head()<prepare_x_and_y>
num_features=len(feature_cols)
 num_targets=len(target_cols )
X = data[:300000]
Xtest = data[300000:]
y = train['target'] 
<train_model>
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()
class SmoothedLogLossMetric(Metric):

 def __init__(self, smoothing=0.001):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0)
 class LogLossMetric(Metric):

 def __init__(self, smoothing=0.0):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0 )
<init_hyperparams>
def process_data(data):
data.loc[:, 'cp_time'] = data.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2, 0: 0, 1: 1, 2: 2})
 data.loc[:, 'cp_dose'] = data.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1, 0: 0, 1: 1})
 return data
def run_training_tabnet(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed, filename="tabnet"):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 loss_fn = LabelSmoothing(0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
if IS_TRAIN:
model.fit(X_train=x_train, y_train=y_train,
eval_set=[(x_valid, y_valid)], eval_metric=[LogLossMetric, SmoothedLogLossMetric],
max_epochs=200, patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=loss_fn 
)
 model.save_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}")
 x_test = test_[feature_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 model.load_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}.model")
 valid_preds = model.predict(x_valid)
 valid_preds = torch.sigmoid(torch.as_tensor(valid_preds)).detach().cpu().numpy()
oof[val_idx] = valid_preds
predictions = model.predict(x_test)
 predictions = torch.sigmoid(torch.as_tensor(predictions)).detach().cpu().numpy()
return oof, predictions
params={'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.04,
'feature_pre_filter': False,
'min_data_in_leaf': 100,
'lambda_l1': 9.486545871986378,
'lambda_l2': 6.887206686817597,
'num_leaves': 230,
'feature_fraction': 0.4,
'bagging_fraction': 1.0,
'bagging_freq': 0,
'min_child_samples': 20}<split>
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training_tabnet(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions
N_FOLDS = 12
oof = np.zeros(len(y))
preds = np.zeros(len(Xtest))
params['learning_rate'] = 0.004
params['num_iterations'] = 12000
i=0
for train_ind, test_ind in tqdm(KFold(n_splits = N_FOLDS,shuffle=True,random_state=21 ).split(X),
total=N_FOLDS, desc="k-fold"):
Xtrain = X.iloc[train_ind]
Xval = X.iloc[test_ind]
ytrain = y.iloc[train_ind]
yval = y.iloc[test_ind]
model = LGBMRegressor(**params)
 model.fit(Xtrain, ytrain,
eval_set =(( Xval,yval)) ,
early_stopping_rounds = 120,
verbose = 1000)
 p = model.predict(Xval)
 oof[test_ind] = p
preds += model.predict(Xtest)/N_FOLDS
i+=1
if i>5:
print(f'MSE::: {np.round(mean_squared_error(y, oof, squared=False),8)}')
 else:
pass<save_to_csv>
train.to_pickle(f"{INT_DIR}/{NB}_pre_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_pre_test.pkl" )
submission = test[['id']]
submission['target'] = preds
submission.to_csv('submission.csv', index = False )<load_from_csv>
run_seeds(train, test, feature_cols, target_cols, NFOLDS, NSEEDS )
df = pd.read_csv(".. /input/remove-trends-giba-explained/train_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True)
 test_df = pd.read_csv(".. /input/remove-trends-giba-explained/test_clean_giba.csv" ).sort_values("time" ).reset_index(drop=True )<count_values>
train.to_pickle(f"{INT_DIR}/{NB}_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test.pkl" )
multiprocessing.cpu_count()<count_values>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
df["group"] = np.arange(df.shape[0])//500_000
df["mg"] = np.arange(df.shape[0])//100_000
df["group"].value_counts()<feature_engineering>
sub6 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub6.to_csv('submission.csv', index=False )
df["category"] = 0
test_df["category"] = 0
df.loc[2_000_000:2_500_000-1, 'category'] = 1
df.loc[4_500_000:5_000_000-1, 'category'] = 1
test_df.loc[500_000:600_000-1, "category"] = 1
test_df.loc[700_000:800_000-1, "category"] = 1<concatenate>
import glob
TARGET = "open_channels"
aug_df = df[df["group"] == 5].copy()
aug_df["category"] = 1
aug_df["group"] = 10
for col in ["signal", TARGET]:
aug_df[col] += df[df["group"] == 8][col].values
df = df.append(aug_df, sort=False )<groupby>
!python.. /input/markscripts/deepinsight_resnest_lightning_v2_infer.py
sub5 = pd.read_csv('submission_resnest_v2.csv' )
df.groupby("group")["signal"].agg({"mean", "std"} )<choose_model_class>
!python.. /input/markscripts/deepinsight_efficientnet_lightning_v7_b3_infer.py
sub4 = pd.read_csv('./submission_effnet_v7_b3.csv' )
<prepare_x_and_y>
! python.. /input/updatedsimplenn/simpleNN_without_ns_newcv.py
sub3 = pd.read_csv('./submission.csv' )
%%time
NUM_FOLDS = 5
skf = StratifiedKFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 oof_preds = np.zeros(( len(df), 11))
y_test = np.zeros(( len(test_df), 11))
for fold,(train_ind, val_ind)in enumerate(skf.split(df, df["group"])) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
print(fold, len(train_df), len(val_df))
for cat in range(2):
fit_df = train_df[train_df["category"] == cat]
y = fit_df[TARGET].values
y[0] = 0
model = xgb.XGBRFClassifier(
 n_estimators = 40,
learning_rate=1,
subsample=0.50,
colsample_bynode=0.25,
reg_lambda=1e-05,
objective= 'multi:softmax',
num_class= len(np.unique(y)) ,
max_depth = 17,
num_parallel_tree = 1,
tree_method = 'gpu_hist',
n_jobs = 2,
verbosity = 0,
predictor = 'gpu_predictor',
).fit(fit_df[features].values, y)
 pred = model.predict_proba(val_df[val_df["category"] == cat][features].values)
 oof_preds[val_ind[np.where(val_df["category"].values == cat)[0]], :pred.shape[1]] = pred
y_test[np.where(test_df["category"].values == cat)[0], :pred.shape[1]] += model.predict_proba(test_df[test_df["category"] == cat][features].values)/NUM_FOLDS
del model; _=gc.collect() 
<compute_train_metric>
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 !python.. /input/python-scripts-moa/2heads_1836_oldcv.py
sub2 = pd.read_csv('./submission.csv' )
f1_score(df["open_channels"], oof_preds.argmax(axis=1), average="macro" )<count_values>
!python.. /input/python2stagenn/2stageNN_with_ns_oldcv.py
sub1 = pd.read_csv('./submission_2stageNN_with_ns_oldcv_0.01822.csv' )
test_df[TARGET] = y_test.argmax(axis=1)
 test_df.iloc[:600_000][TARGET].value_counts() /600_000<save_to_csv>
!python.. /input/simplennoldcvfinal/script_NN_836_final.py
sub7 = pd.read_csv('submission_script_simpleNN_oldcv_0.01836.csv' )
<import_modules><EOS> 
submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission.iloc[:, 1:] = 0
submission.iloc[:, 1:] =(sub1.iloc[:,1:]*0.44 + sub7.iloc[:,1:]*0.15 + sub4.iloc[:,1:]*0.35 +sub2.iloc[:,1:]*0.06)*0.78 + sub6.iloc[:,1:]*0.1 + sub7.iloc[:,1:]*0.05 + sub5.iloc[:,1:]*0.07
submission.to_csv('submission.csv', index=False )