Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<categorify>
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 sys.path.append('.. /input/rank-gauss')
 simplefilter(action='ignore', category=FutureWarning)
 sns.set()
calendar.fillna('0', inplace=True)
 label_encoder = LabelEncoder()
label_cols = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for col in label_cols:
calendar[col] = label_encoder.fit_transform(calendar[col])
 calendar.head(5 )<feature_engineering>
data_path = '.. /input/lish-moa/'
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.7
decompo = 'PCA'
ncompo_genes = 80
ncompo_cells = 10
encoding = 'dummy'
base_seed = 2020
calendar['is_weekend'] = calendar['wday'].apply(lambda x: 1 if x == 1 or x == 2 else 0)
 seasons = {1: 1, 2: 1, 12: 1, 3: 2, 4: 2, 5: 2, 6: 3, 7: 3, 8: 3, 9: 4, 10: 4, 11: 4 }
calendar['season'] = calendar['month'].apply(lambda x: seasons[x])
 calendar.head()<data_type_conversions>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
calendar['wm_yr_wk'] = calendar['wm_yr_wk'].astype(np.int16)
 calendar['wday'] = calendar['wday'].astype(np.int8)
 calendar['month'] = calendar['month'].astype(np.int8)
 calendar['year'] = calendar['year'].astype(np.int16)
 calendar['snap_CA'] = calendar['snap_CA'].astype(np.int8)
 calendar['snap_TX'] = calendar['snap_TX'].astype(np.int8)
 calendar['snap_WI'] = calendar['snap_WI'].astype(np.int8)
 calendar['is_weekend'] = calendar['is_weekend'].astype(np.int8)
 calendar['season'] = calendar['season'].astype(np.int8)
 calendar['event_name_1'] = calendar['event_name_1'].astype(np.int16)
 calendar['event_type_1'] = calendar['event_type_1'].astype(np.int8)
 calendar['event_name_2'] = calendar['event_name_2'].astype(np.int8)
 calendar['event_type_2'] = calendar['event_type_2'].astype(np.int8 )<load_from_csv>
seed_everything(base_seed )
train_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 train_data.head()<categorify>
test = pd.read_csv(data_path + 'test_features.csv')
 train = pd.read_csv(data_path + 'train_features.csv')
 targets = pd.read_csv(data_path + 'train_targets_scored.csv')
 if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True )
def melt_item_group(group):
numcols = [f"d_{day}" for day in range(0,1941 + 1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
for day in range(1942, 1942 + 28):
group[f"d_{day}"] = np.nan
dt = pd.melt(
 group,
id_vars = catcols,
value_vars = [col for col in group.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales" 
)
 return dt<data_type_conversions>
data_all = pd.concat([train, test], ignore_index=True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
prices = pd.read_csv('.. /input/m5-forecasting-accuracy/sell_prices.csv')
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices['sell_price'] = prices['sell_price'].astype(np.float32)
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices.set_index(['store_id', 'item_id', 'wm_yr_wk'], inplace=True)
 prices = prices.sort_index()<data_type_conversions>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == 'boxcox':
print('boxcox')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == 'norm':
print('norm')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis=0)
 elif scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass
def convert_categorical_columns(df, columns):
for column in columns:
df[column] = df[column].astype('category')
 df[column] = df[column].cat.codes
df[column] = df[column].astype('category' )<prepare_x_and_y>
if decompo == 'PCA':
print('PCA')
 GENES = [col for col in data_all.columns if col.startswith('g-')]
CELLS = [col for col in data_all.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis=1)
 else:
pass
def get_dataset_simple(df, label_column):
test_df = df.loc['d_1914':,:].copy()
valid_df = df.loc['d_1914': 'd_1941',:].copy()
train_df = df.loc[df.index[0]: 'd_1913',:].copy()
train_df.loc[:,label_column] = train_df.loc[:,label_column]
return train_df, valid_df, test_df<define_variables>
if encoding == 'lb':
print('Label Encoding')
 for feat in ['cp_time', 'cp_dose']:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == 'dummy':
print('One-hot')
 data_all = pd.get_dummies(data_all, columns=['cp_time', 'cp_dose'] )
def get_dataset(df, label_column):
test_df = df.loc['d_1914':,:].copy()
df_shape = df.loc[:'d_1941',:].shape
samples_idx = random.sample(range(df_shape[0]), int(0.2 * df_shape[0]))
valid_idxs = samples_idx
train_idxs = list(set(list(range(0, df_shape[0])))- set(samples_idx))
return train_idxs, valid_idxs, test_df<load_from_csv>
GENES = [col for col in data_all.columns if col.startswith('g-')]
CELLS = [col for col in data_all.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
data_all['g_'+stats] = getattr(data_all[GENES], stats )(axis=1)
 data_all['c_'+stats] = getattr(data_all[CELLS], stats )(axis=1)
 data_all['gc_'+stats] = getattr(data_all[GENES+CELLS], stats )(axis=1)
evaluation_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 evaluation_data.loc[:,'id'] = evaluation_data['id'].apply(lambda x: x[:-10] + 'validation')
 evaluation_data.set_index('id', inplace=True)
 evaluation_data = evaluation_data.loc[:,'d_1914': 'd_1941'].copy()
def get_rmse(submission_validation):
sub = submission_validation.set_index('id')
 error = mean_squared_error(sub.values, evaluation_data.loc[sub.index,:].values)
 return error<feature_engineering>
with open('data_all.pickle', 'wb')as f:
pickle.dump(data_all, f )
def split_prediction(prediction):
columns = ['id'] + ['F%s' % i for i in range(1, 29)]
prediction = prediction.reset_index().set_index('id')
 prediction_evaluation = prediction.loc[:,'d_1914':'d_1941'].reset_index().copy()
prediction_evaluation.columns = columns
prediction_validation = prediction.loc[:,'d_1942':].reset_index().copy()
prediction_validation.columns = columns
prediction_validation.loc[:, 'id'] = prediction_validation['id'].str[:-10] + 'validation'
return prediction_validation, prediction_evaluation<data_type_conversions>
with open('data_all.pickle', 'rb')as f:
data_all = pickle.load(f )
def create_price_features(group_df, horizon=28):
lag = horizon // 7
group_df['sell_price_diff_shift_1'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.shift(1)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.shift(horizon)).astype(np.float32)
 group_df['sell_price_diff_rolling_7'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.rolling(7 ).mean() ).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_shift_1'] = group_df.groupby('id')['sell_price_diff_shift_1'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_shift_{horizon}'] = group_df.groupby('id')[f'sell_price_diff_shift_{horizon}'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_rolling_7'] = group_df.groupby('id')['sell_price_diff_rolling_7'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}'] = group_df.groupby('id')['sell_price_diff_rolling_7'].transform(lambda x: x - x.shift(horizon)).astype(np.float32 )<define_variables>
features_todrop = ['sig_id', 'cp_type']
data_all.drop(features_todrop, axis=1, inplace=True)
 try:
targets.drop('sig_id', axis=1, inplace=True)
 except:
pass
train_df = data_all[:train.shape[0]]
train_df.reset_index(drop=True, inplace=True)
 train_df = pd.concat([train_df, targets], axis=1)
 test_df = data_all[train_df.shape[0]:]
test_df.reset_index(drop=True, inplace=True )
def get_created_features(horizon=28):
feature_columns = [
'sell_price_diff_shift_1',
'sell_price_diff_rolling_7',
f'sell_price_diff_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_shift_1',
f'sell_price_diff_shift_{horizon}_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_rolling_7',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
]
return feature_columns<define_variables>
device =('cuda' if torch.cuda.is_available() else 'cpu')
 hsize = 1024
dropratio = 0.2
batchsize = 128
lr = 0.001
wd = 1e-5
smoothing = 0.001
p_min = smoothing
p_max = 1 - smoothing
nepoch = 20
earlystop = True
earlystop_step = 10
lr_scheduler = 'OneCycleLR'
pct_start = 0.1
div_factor = 1e3
factor=0.5
patience=3
nseed = 5
nfold = 7
eval_strategy = 'kfold'
%%time
idx_feature = ['id']
categorical_feature = [
'dept_id' ,
'state_id',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
feature_columns = categorical_feature + [
'sales_shift_28',
'sales_mean_rolling_4_wday_shift_4',
'sales_mean_rolling_4_wday_shift_8',
'sell_price',
]
label_column = 'sales'
dummy_subm = pd.DataFrame(columns=['id'] + ['F%s' % i for i in range(1, 29)])
 submission_validation = dummy_subm.copy()
submission_evaluation = dummy_subm.copy()
submission_validation_lgb = dummy_subm.copy()
submission_evaluation_lgb = dummy_subm.copy()
groups = train_data.groupby('store_id')
 len_group = len(groups)
 for i,(store_id, group)in enumerate(groups):
print(store_id)
 group = melt_item_group(group)
 group_df = group.join(calendar.set_index('d'), on='d')
 del group
group_df.sales.fillna(0, inplace=True)
 group_df = group_df.set_index(['d'])
 group_df['sales_shift_28'] = group_df.groupby(['id'])['sales'].transform(lambda x: x.shift(28)).astype(np.float32)
 group_df['sales_mean_rolling_4_wday'] = group_df.groupby(['id', 'wday'])['sales'].transform(lambda x: x.rolling(4 ).mean() ).astype(np.float32)
 group_df['sales_mean_rolling_4_wday_shift_4'] = group_df.groupby(['id', 'wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(4)).astype(np.float32)
 group_df['sales_mean_rolling_4_wday_shift_8'] = group_df.groupby(['id', 'wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(8)).astype(np.float32)
 print('add prices')
 group_df = group_df.join(
 prices.loc[store_id], on=['item_id', 'wm_yr_wk'] 
)
 label_cols_prices = ['state_id', 'dept_id']
convert_categorical_columns(group_df, label_cols_prices)
 create_price_features(group_df)
 feature_columns += get_created_features()
group_df.dropna(inplace=True)
 print('drop rows with na')
 train_idxs, valid_idxs, test_df = get_dataset(
 group_df[feature_columns + [label_column] + idx_feature],
label_column=label_column 
)
 print('get_dataset_simple')
 prediction = test_df.reset_index().set_index(['d', 'id',])['sales_mean_rolling_4_wday_shift_4']
prediction = prediction.unstack(level=0)
 prediction_validation, prediction_evaluation = split_prediction(prediction)
 try:
dtrain = lgb.Dataset(group_df.iloc[train_idxs][feature_columns], label=group_df.iloc[train_idxs][label_column], categorical_feature=categorical_feature)
 dvalid = lgb.Dataset(group_df.iloc[valid_idxs][feature_columns], label=group_df.iloc[valid_idxs][label_column], categorical_feature=categorical_feature)
 param = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 1024,
'min_data_in_leaf': 1024,
'feature_fraction': 0.2,
'max_bin': 10,
'boost_from_average': False,
'verbose': -1,
}
bst = lgb.train(param, dtrain, valid_sets=[dvalid], num_boost_round = 2000, early_stopping_rounds=30, verbose_eval=True, categorical_feature=categorical_feature)
 prediction_lgb = bst.predict(test_df.reset_index().set_index(['d', 'id',])[feature_columns])
 prediction_lgb_df = test_df.reset_index() [['d', 'id']].copy()
prediction_lgb_df['prediction'] = prediction_lgb
prediction_lgb_df = prediction_lgb_df.set_index(['d', 'id',] ).unstack(level=0 ).reset_index()
prediction_lgb_df.columns = ['id'] + ['d_{}'.format(i)for i in range(1914, 1970)]
prediction_validation_lgb, prediction_evaluation_lgb = split_prediction(prediction_lgb_df)
 except Exception as e:
print(e)
 prediction_validation_lgb = prediction_validation
prediction_evaluation_lgb = prediction_evaluation
finally:
submission_validation_lgb = submission_validation_lgb.append(prediction_validation_lgb)
 submission_evaluation_lgb = submission_evaluation_lgb.append(prediction_evaluation_lgb)
 submission_validation = submission_validation.append(prediction_validation)
 submission_evaluation = submission_evaluation.append(prediction_evaluation )<save_to_csv>
class Model(nn.Module):
def __init__(self, n_features, n_targets, hidden_size=512, dropratio=0.2):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(n_features)
 self.dropout1 = nn.Dropout(dropratio)
 self.dense1 = nn.utils.weight_norm(nn.Linear(n_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropratio)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropratio)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, n_targets))
self.relu = nn.ReLU()
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = self.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = self.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
submission = submission_validation.append(submission_evaluation)
 submission.to_csv('/kaggle/working/submission_mean.csv', index=False)
 submission_lgb = submission_validation_lgb.append(submission_evaluation_lgb)
 submission_lgb.to_csv('/kaggle/working/submission_lgb.csv', index=False )<import_modules>
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets, n_classes, smoothing=0.0):
assert 0 <= smoothing <= 1
with torch.no_grad() :
targets = targets *(1 - smoothing)+ torch.ones_like(targets ).to(device)* smoothing / n_classes
return targets
def forward(self, inputs, targets):
targets = SmoothCrossEntropyLoss()._smooth(targets, inputs.shape[1], self.smoothing)
 if self.weight is not None:
inputs = inputs * self.weight.unsqueeze(0)
 loss = F.binary_cross_entropy_with_logits(inputs, targets)
 return loss
import pandas as pd
import numpy as np
import json
import os
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras.utils.vis_utils import plot_model
import tensorflow.keras.layers as L
import keras.backend as K
import tensorflow as tf<install_modules>
def running_train(X_train, Y_train, X_val, Y_val, dataloader, i_fold=None, seed=None):
model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 criterion = SmoothCrossEntropyLoss(smoothing=smoothing)
 metric = lambda inputs, targets : F.binary_cross_entropy(( torch.clamp(torch.sigmoid(inputs), p_min, p_max)) , targets)
 optimizer = torch.optim.Adam(params=model.parameters() , lr=lr, weight_decay=wd)
 if lr_scheduler == 'OneCycleLR' or lr_scheduler == 'both':
scheduler1 = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=pct_start, div_factor=div_factor,
max_lr=1e-2, epochs=nepoch, steps_per_epoch=len(dataloader))
if lr_scheduler == 'ReduceLROnPlateau' or lr_scheduler == 'both':
scheduler2 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.5, patience=3)
 min_valmetric = np.inf
step = 0
for epoch in range(nepoch):
train_loss = 0
train_metric = 0
for i,(X, Y)in enumerate(dataloader):
model.train()
predictions = model(X.to(device=device))
loss = criterion(predictions, Y.to(device=device))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if lr_scheduler == 'OneCycleLR' or lr_scheduler == 'both':
scheduler1.step()
train_loss += loss.item()
train_metric += metric(predictions, Y.to(device=device))
train_loss /= len(dataloader)
 train_metric /= len(dataloader)
 model.eval()
predictions = model(X_val.to(device=device))
val_loss = criterion(predictions, Y_val.to(device=device))
val_metric = metric(predictions, Y_val.to(device=device))
if lr_scheduler == 'ReduceLROnPlateau' or lr_scheduler == 'both':
scheduler2.step(val_metric)
 print('Epoch {}/{}, Train Loss={:5f}, Train Metric={:.5f}, Val Loss={:.5f}, Val Metric={:.5f}'.format(
 epoch + 1, nepoch, train_loss, train_metric, val_loss, val_metric))
if val_metric.item() < min_valmetric:
min_valmetric = val_metric.item()
model_name = 'model_{}_{}.pth'.format(i_fold + 1, seed)if eval_strategy == 'kfold' else 'model_single.pth'
torch.save(model.state_dict() , model_name)
 elif earlystop:
step += 1
if step > earlystop_step:
break
!pip install spektral -q<import_modules>
def train_kfold_model(train_df):
X_train_val = torch.from_numpy(train_df.iloc[:, :-targets.shape[1]].values ).to(dtype=torch.float32)
 Y_train_val = torch.from_numpy(train_df.iloc[:, -targets.shape[1]:].values ).to(dtype=torch.float32)
 X_test = torch.from_numpy(test_df.values ).to(dtype=torch.float32)
 oof = torch.zeros(X_train_val.shape[0], Y_train_val.shape[1])
 prediction_test = torch.zeros(X_test.shape[0], Y_train_val.shape[1])
 for i_seed in range(nseed):
seed = random.randint(0, base_seed)
 seed_everything(seed)
 print('Seed: {}, {}/{}'.format(seed, i_seed + 1, nseed))
mskf = MultilabelStratifiedKFold(n_splits=nfold, random_state=seed, shuffle=True)
 for i_fold,(train_idx, val_idx)in enumerate(mskf.split(X_train_val, Y_train_val)) :
print("
X_train, Y_train = X_train_val[train_idx], Y_train_val[train_idx]
X_val, Y_val = X_train_val[val_idx], Y_train_val[val_idx]
dataset = TensorDataset(X_train, Y_train)
 dataloader = DataLoader(dataset, batch_size=batchsize, shuffle=True)
 running_train(X_train, Y_train, X_val, Y_val, dataloader, i_fold=i_fold, seed=seed)
 print('predict on oof...', end='')
 model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 model.load_state_dict(torch.load('model_{}_{}.pth'.format(i_fold + 1, seed)))
 model.eval()
oof[val_idx] += torch.clamp(torch.sigmoid(model(X_val.to(device=device)).detach().cpu()), p_min, p_max)/ nseed
print(' done.')
 print('predict on test...', end='')
 prediction_test += torch.clamp(torch.sigmoid(model(X_test.to(device=device)).detach().cpu()), p_min, p_max)/(nfold * nseed)
 print(' done.
 ')
 cv_score = F.binary_cross_entropy(oof, Y_train_val)
 print('{} folds cv_score: {:.5f}'.format(nfold, cv_score))
return oof, pd.DataFrame(prediction_test.numpy() , columns=targets.columns )
from spektral.layers import GraphConv<load_from_disk>
def train_single_model(train_df):
train_df, val_df = train_test_split(train_df, test_size=0.2, shuffle=True, random_state=base_seed)
 X_train = train_df.iloc[:, :-targets.shape[1]].values
Y_train = train_df.iloc[:, -targets.shape[1]:].values
X_val = val_df.iloc[:, :-targets.shape[1]].values
Y_val = val_df.iloc[:, -targets.shape[1]:].values
X_test = test_df.values
X_train = torch.from_numpy(X_train ).to(dtype=torch.float32)
 Y_train = torch.from_numpy(Y_train ).to(dtype=torch.float32)
 X_val = torch.from_numpy(X_val ).to(dtype=torch.float32)
 Y_val = torch.from_numpy(Y_val ).to(dtype=torch.float32)
 X_test = torch.from_numpy(X_test.astype(np.float32))
dataset = TensorDataset(X_train, Y_train)
 dataloader = DataLoader(dataset, batch_size=batchsize, shuffle=True)
 running_train(X_train, Y_train, X_val, Y_val, dataloader)
 print('predict on test...', end='')
 model = Model(n_features=X_train.shape[1], n_targets=Y_train.shape[1], hidden_size=hsize, dropratio=dropratio ).to(device)
 model.load_state_dict(torch.load('model_single.pth'))
model.eval()
predictions = torch.clamp(torch.sigmoid(model(X_test.to(device)).detach().cpu()), p_min, p_max)
 print(' done.
 ')
 return pd.DataFrame(predictions.numpy() , columns=targets.columns )
train_json_path = "/kaggle/input/stanford-covid-vaccine/train.json"
test_json_path = "/kaggle/input/stanford-covid-vaccine/test.json"
sample_sub_path = "/kaggle/input/stanford-covid-vaccine/sample_submission.csv"
output_path = "./"
bpps_path = "/kaggle/input/stanford-covid-vaccine/bpps"
train_df = pd.read_json(train_json_path, lines=True)
 test_df = pd.read_json(test_json_path, lines=True)
 public_df = test_df.query("seq_length == 107" ).copy()
private_df = test_df.query("seq_length == 130" ).copy()<prepare_x_and_y>
if eval_strategy == 'kfold':
oof, test_pred = train_kfold_model(train_df)
 elif eval_strategy == 'single':
test_pred = train_single_model(train_df)
 else:
print('eval_strategy should be "kfold" or "single"' )
train_y = np.array(train_df[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
train_y.shape<categorify>
test = pd.read_csv(data_path + 'test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 test_pred['sig_id'] = sig_id
sub = pd.merge(test[['sig_id']], test_pred, on='sig_id', how='left')
 sub.fillna(0, inplace=True)
 sub.to_csv('submission.csv', index=False )