Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<count_values>

 BaseShuffleSplit, _validate_shuffle_split
def IterativeStratification(labels, r, random_state):

 n_samples = labels.shape[0]
test_folds = np.zeros(n_samples, dtype=int)
 c_folds = r * n_samples
c_folds_labels = np.outer(r, labels.sum(axis=0))
labels_not_processed_mask = np.ones(n_samples, dtype=bool)
 while np.any(labels_not_processed_mask):
num_labels = labels[labels_not_processed_mask].sum(axis=0)
 if num_labels.sum() == 0:
sample_idxs = np.where(labels_not_processed_mask)[0]
for sample_idx in sample_idxs:
fold_idx = np.where(c_folds == c_folds.max())[0]
if fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
c_folds[fold_idx] -= 1
break
label_idx = np.where(num_labels == num_labels[np.nonzero(num_labels)].min())[0]
if label_idx.shape[0] > 1:
label_idx = label_idx[random_state.choice(label_idx.shape[0])]
sample_idxs = np.where(np.logical_and(labels[:, label_idx].flatten() , labels_not_processed_mask)) [0]
for sample_idx in sample_idxs:
label_folds = c_folds_labels[:, label_idx]
fold_idx = np.where(label_folds == label_folds.max())[0]
if fold_idx.shape[0] > 1:
temp_fold_idx = np.where(c_folds[fold_idx] ==
c_folds[fold_idx].max())[0]
fold_idx = fold_idx[temp_fold_idx]
if temp_fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(temp_fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
labels_not_processed_mask[sample_idx] = False
c_folds_labels[fold_idx, labels[sample_idx]] -= 1
c_folds[fold_idx] -= 1
return test_folds
class MultilabelStratifiedKFold(_BaseKFold):

 def __init__(self, n_splits=3, shuffle=False, random_state=None):
super(MultilabelStratifiedKFold, self ).__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
 def _make_test_folds(self, X, y):
y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(type_of_target_y))
num_samples = y.shape[0]
rng = check_random_state(self.random_state)
 indices = np.arange(num_samples)
 if self.shuffle:
rng.shuffle(indices)
 y = y[indices]
r = np.asarray([1 / self.n_splits] * self.n_splits)
 test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 return test_folds[np.argsort(indices)]
def _iter_test_masks(self, X=None, y=None, groups=None):
test_folds = self._make_test_folds(X, y)
 for i in range(self.n_splits):
yield test_folds == i
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedKFold, self ).split(X, y, groups)
 class RepeatedMultilabelStratifiedKFold(_RepeatedSplits):

 def __init__(self, n_splits=5, n_repeats=10, random_state=None):
super(RepeatedMultilabelStratifiedKFold, self ).__init__(
 MultilabelStratifiedKFold, n_repeats=n_repeats, random_state=random_state,
n_splits=n_splits)
 class MultilabelStratifiedShuffleSplit(BaseShuffleSplit):

 def __init__(self, n_splits=10, test_size="default", train_size=None,
random_state=None):
super(MultilabelStratifiedShuffleSplit, self ).__init__(
 n_splits=n_splits, test_size=test_size, train_size=train_size, random_state=random_state)
 def _iter_indices(self, X, y, groups=None):
n_samples = _num_samples(X)
 y = check_array(y, ensure_2d=False, dtype=None)
 y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(
 type_of_target_y))
n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
self.train_size)
 n_samples = y.shape[0]
rng = check_random_state(self.random_state)
 y_orig = y.copy()
r = np.array([n_train, n_test])/(n_train + n_test)
 for _ in range(self.n_splits):
indices = np.arange(n_samples)
 rng.shuffle(indices)
 y = y_orig[indices]
test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 test_idx = test_folds[np.argsort(indices)] == 1
test = np.where(test_idx)[0]
train = np.where(~test_idx)[0]
yield train, test
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedShuffleSplit, self ).split(X, y, groups )
ts.groupby(["DayOfWeek"])["Hour"].value_counts()<prepare_x_and_y>
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
train_full = ts.copy()
X_train_full, y_train_full = np.array(train_full[["DayOfWeek", "PdDistrict", "X", "Y", "Hour"]]), np.array(train_full[["Category"]])
 y_train_full = y_train_full.ravel()
X_test = test_set_full_raw.copy()
X_test["Hour"] = X_test.Dates.apply(lambda date_string: date_string[11:-6])
 X_test = X_test.drop(columns=["Dates", "Address"])
 X_test = np.array(X_test )<split>
TEST_FEATURES_PATH = "/kaggle/input/lish-moa/test_features.csv"
TRAIN_FEATURES_PATH = "/kaggle/input/lish-moa/train_features.csv"
TRAIN_TARGETS_PATH = "/kaggle/input/lish-moa/train_targets_scored.csv"
TRAIN_TARGETS_NONSCORED_PATH = "/kaggle/input/lish-moa/train_targets_nonscored.csv"
SAMPLE_SUB_PATH = "/kaggle/input/lish-moa/sample_submission.csv"
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.25, random_state=42)
 for train_index, test_index in sss.split(X_train_full, y_train_full):
X_train, y_train = X_train_full[train_index], y_train_full[train_index]
X_val, y_val = X_train_full[test_index], y_train_full[test_index]<categorify>
test_features_df = pd.read_csv(TEST_FEATURES_PATH ).sort_values(by='sig_id')
 train_features_df = pd.read_csv(TRAIN_FEATURES_PATH ).sort_values(by='sig_id')
 train_targets_df = pd.read_csv(TRAIN_TARGETS_PATH ).sort_values(by='sig_id')
 train_targets_nonscored_df = pd.read_csv(TRAIN_TARGETS_NONSCORED_PATH)
 sample_sub_df = pd.read_csv(SAMPLE_SUB_PATH ).sort_values(by='sig_id' )
num_attribs = [2,3]
cat_attribs = [0,1,4]
num_pipeline = Pipeline([ 
('std_scaler', StandardScaler())
 ])
 full_pipeline = ColumnTransformer([ 
('num', num_pipeline, num_attribs), 
('cat', OneHotEncoder() , cat_attribs)
 ])
 X_train_prepared = full_pipeline.fit_transform(X_train)
 X_val_prepared = full_pipeline.transform(X_val)
 X_test_prepared = full_pipeline.transform(X_test )<import_modules>
train_targets_df['5-alpha_reductase_inhibitor'].value_counts()
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier,VotingClassifier
import xgboost<split>
value_counts_arr = np.sort([train_targets_df[col].value_counts() [1] for col in train_targets_df.columns])
 print(value_counts_arr )
ss = StratifiedShuffleSplit(n_splits=1, train_size=100_000, random_state=42)
 for train_index, _ in ss.split(X_train_prepared, y_train):
X_train_prepared_small, y_train_small = X_train_prepared[train_index], y_train[train_index].ravel()
X_train_prepared_small.shape, y_train_small.shape<train_on_grid>
rf_clf = RandomForestClassifier(max_depth=16, random_state=42, n_jobs=-1, verbose=3)
 xg_clf = xgboost.XGBClassifier()
estimators = [ 
("rf", rf_clf), 
("xg", xg_clf)
 ]
voting_clf = VotingClassifier(estimators, n_jobs=-1, voting="soft")
 voting_clf.fit(X_train_prepared_small, y_train_small)
 voting_clf.score(X_val_prepared, y_val )<save_to_csv>
y_pred = voting_clf.predict_proba(X_test_prepared)
 pred_df = pd.DataFrame(y_pred, columns=[voting_clf.classes_])
 pred_df["Id"]= list(range(pred_df.shape[0]))
pred_df.to_csv("crime_pred_02.zip", compression="zip", index=False)
 pred_df.head()<load_from_csv>
X = train_features_df.drop(columns=['sig_id'])
 X_test = test_features_df.drop(columns=['sig_id'])
 y = train_targets_df.drop(columns=['sig_id'] )
train = pd.read_csv('/kaggle/input/sf-crime/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv', parse_dates=['Dates'] )<feature_engineering>
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(X[genes+cells] )
train['year'] = train['Dates'].dt.year
train['month'] = train['Dates'].dt.month
train['day'] = train['Dates'].dt.day
train['DayOfWeek'] = train['Dates'].dt.weekday
train['hour'] = train['Dates'].dt.hour
train['minute'] = train['Dates'].dt.minute<feature_engineering>
X.loc[:,genes+cells] = transformer.transform(X[genes+cells])
 X_test.loc[:,genes+cells] = transformer.transform(X_test[genes+cells])
train['Block'] = train['Address'].str.contains('Block', case=False)
 train.head()<feature_engineering>
from sklearn.decomposition import PCA
train['AV'] = train['Address'].str.contains('AV', case=False)
 train.head()<feature_engineering>
def pca_number(X_genes, X_test_genes, X_cells, X_test_cells, X, X_test, n_gene=200, n_cells=10):
pca = PCA(n_components=n_gene)
 data = pca.fit_transform(pd.concat([X_genes,X_test_genes]))
X_temp = pd.DataFrame(data=data, columns = ['pca_gene'+str(i)for i in range(n_gene)])
 X_genes = X_temp[:len(X_genes)]
X_test_genes = X_temp[len(X_genes):].reset_index(drop=True)
 pca = PCA(n_components=n_cells)
 data = pca.fit_transform(pd.concat([X_cells,X_test_cells]))
X_temp = pd.DataFrame(data=data, columns = ['pca_cells'+str(i)for i in range(n_cells)])
 X_cells = X_temp[:len(X_cells)]
X_test_cells = X_temp[len(X_cells.index):].reset_index(drop=True)
 New_x = pd.concat([X, X_genes, X_cells], axis=1)
 New_xtest = pd.concat([X_test, X_test_genes, X_test_cells], axis=1)
 return New_x, New_xtest
train['ST'] = train['Address'].str.contains('ST', case=False)
 train.head()<feature_engineering>
features = X.columns
genes = [col for col in features if col.startswith('g-')]
test['year'] = test['Dates'].dt.year
test['month'] = test['Dates'].dt.month
test['day'] = test['Dates'].dt.day
test['DayOfWeek'] = test['Dates'].dt.weekday
test['hour'] = test['Dates'].dt.hour
test['minute'] = test['Dates'].dt.minute<feature_engineering>
cells = [col for col in features if col.startswith('c-')]
test['Block'] = test['Address'].str.contains('Block', case=False)
 test.head()<feature_engineering>
n_genes=35
n_cells=5
kmeans = KMeans(n_genes)
 data = kmeans.fit(X[genes].append(X_test[genes]))
X['kmeans_g'] = data.labels_[:X.shape[0]]
X_test['kmeans_g'] = data.labels_[X.shape[0]:]
kmeans = KMeans(n_cells)
 data = kmeans.fit(X[cells].append(X_test[cells]))
X['kmeans_c'] = data.labels_[:X.shape[0]]
X_test['kmeans_c'] = data.labels_[X.shape[0]:]
test['AV'] = test['Address'].str.contains('AV', case=False)
 test.head()<feature_engineering>
for df in X, X_test:
df['g_sum'] = df[genes].sum(axis = 1)
 df['g_mean'] = df[genes].mean(axis = 1)
 df['g_std'] = df[genes].std(axis = 1)
 df['g_kurt'] = df[genes].kurtosis(axis = 1)
 df['g_skew'] = df[genes].skew(axis = 1)
 df['c_sum'] = df[cells].sum(axis = 1)
 df['c_mean'] = df[cells].mean(axis = 1)
 df['c_std'] = df[cells].std(axis = 1)
 df['c_kurt'] = df[cells].kurtosis(axis = 1)
 df['c_skew'] = df[cells].skew(axis = 1)
 df['gc_sum'] = df[genes + cells].sum(axis = 1)
 df['gc_mean'] = df[genes + cells].mean(axis = 1)
 df['gc_std'] = df[genes + cells].std(axis = 1)
 df['gc_kurt'] = df[genes + cells].kurtosis(axis = 1)
 df['gc_skew'] = df[genes + cells].skew(axis = 1)
 df['g_sum_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).sum().values.sum(axis=1))))
df['g_mean_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).mean().values.mean(axis=1))))
df['g_std_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).std().values.std(axis=1))))
df['c_sum_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).sum().values.sum(axis=1))))
df['c_mean_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).mean().values.mean(axis=1))))
df['c_std_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).std().values.std(axis=1))))
test['ST'] = test['Address'].str.contains('ST', case=False)
 test.head()<groupby>
X = pd.get_dummies(X, columns=['cp_dose', 'cp_time'])
 X_test = pd.get_dummies(X_test, columns=['cp_dose', 'cp_time'] )
train.groupby(['Category'])['AV'].mean()<groupby>
X_genes = X[genes].copy()
X_test_genes = X_test[genes].copy()
X_cells = X[cells].copy()
X_test_cells = X_test[cells].copy()
train.groupby(['Category'])['ST'].mean()<count_values>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
train['Category'].value_counts()<prepare_x_and_y>
X, X_test = pca_number(X_genes, X_test_genes, X_cells, X_test_cells, X, X_test, n_gene=650, n_cells=50 )
y = train['Category']<categorify>
control_mask = test_features_df['cp_type']=='ctl_vehicle'
le = LabelEncoder()
le.fit(train['PdDistrict'])
 le.classes_<categorify>
train['PdDistrict'] = le.transform(train['PdDistrict'] )<categorify>
X = X.drop(['cp_type'], axis=1)
 X_test = X_test.drop(['cp_type'], axis=1 )
test['PdDistrict'] = le.transform(test['PdDistrict'])
 test.head()<count_unique_values>
var_thresh = VarianceThreshold(0.8)
 data = X.append(X_test)
 var_thresh.fit(data )
train.Address.nunique()<count_unique_values>
cats = ['cp_dose_D1','cp_dose_D2','cp_time_24','cp_time_48','cp_time_72']
test.Address.nunique()<train_model>
extras = ['kmeans_g', 'kmeans_c','g_sum', 'g_mean', 'g_std', 'g_kurt','g_skew', 'c_sum','c_mean', 'c_std', 'c_kurt', 'c_skew', 'gc_sum','gc_mean','gc_std','gc_kurt','gc_skew','g_sum_cp_dose','g_mean_cp_dose','g_std_cp_dose','c_sum_cp_dose', 'c_mean_cp_dose', 'c_std_cp_dose']
le.fit(list(train['Address'])+ list(test['Address']))<categorify>
sum(selected_columns )
train['Address'] = le.transform(train['Address'])
 test['Address'] = le.transform(test['Address'] )<feature_engineering>
selected_columns[872:900] = [True]*28
train['Dates'].dt.week<feature_engineering>
t = X.loc[:, selected_columns]
t1 = X_test.loc[:, selected_columns]
test['Dates'].dt.week<prepare_x_and_y>
X = X.loc[:, selected_columns]
X_test = X_test.loc[:, selected_columns]
y = train['Category']
train = train.drop(['Dates', 'Resolution', 'Descript', 'Category'], axis=1)
 test = test.drop(['Id', 'Dates'], axis=1 )<choose_model_class>
!pip install transformers &> /dev/null
!git clone https://github.com/fastai/fastai &> /dev/null
!pip install -e "fastai[dev]" &> /dev/null
!pip install fastai2 &> /dev/null
<train_model>
from fastai.tabular.all import *
from fastai.tabular.data import *
cat = CatBoostClassifier(task_type='GPU')
 cat.fit(train, y )<predict_on_test>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.25)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
preds = cat.predict_proba(test )<load_from_csv>
class MoaDataset(torch.utils.data.Dataset):
def __init__(self, X, y):
self.X = X.values
self.y = y.astype(float)
 def __getitem__(self, idx):
return(torch.tensor(self.X[idx], dtype=torch.float), torch.tensor(self.y[idx], dtype=torch.float))
def __len__(self):
return len(self.y )
sample = pd.read_csv('/kaggle/input/sf-crime/sampleSubmission.csv')
 sample.head()<prepare_output>
class MoaTestDataset(torch.utils.data.Dataset):
def __init__(self, X):
self.X = X.values
self.y = np.zeros(len(X))
def __getitem__(self, idx):
return(torch.tensor(self.X[idx], dtype=torch.float), torch.tensor(self.y[idx], dtype=torch.float))
def __len__(self):
return len(self.X )
sample.iloc[:,1:] = preds
sample.head()<save_to_csv>
@delegates(torch.optim.AdamW.__init__)
 def pytorch_adamw(param_groups, **kwargs):
return OptimWrapper(torch.optim.AdamW([{'params': ps, **kwargs} for ps in param_groups]))
sample.to_csv("baseline_lgbm.csv", index=False )<train_model>
CustomAdamW =partial(pytorch_adamw, lr=1e-3, weight_decay=1e-5 )
<import_modules>
y = y.to_numpy()
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import lightgbm as lgb<load_from_csv>
from torch.nn.modules.loss import _WeightedLoss
import torch.nn.functional as F
train_data = pd.read_csv('.. /input/sf-crime/train.csv.zip', parse_dates=['Dates'])
 test_data = pd.read_csv('.. /input/sf-crime/test.csv.zip', parse_dates=['Dates'] )<feature_engineering>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
all_features = pd.concat(( train_data.iloc[:, [0, 3, 4, 6, 7, 8]],
test_data.iloc[:, [1, 2, 3, 4, 5, 6]]),
sort=False)
 num_train = train_data.shape[0]
train_labels = pd.get_dummies(train_data['Category'] ).values
num_outputs = train_labels.shape[1]
train_labels = np.argmax(train_labels, axis=1)
 all_features['year'] = all_features.Dates.dt.year
all_features['month'] = all_features.Dates.dt.month
all_features['new_year'] = all_features['month'].apply(
 lambda x: 1 if x == 1 or x == 2 else 0)
 all_features['day'] = all_features.Dates.dt.day
all_features['hour'] = all_features.Dates.dt.hour
all_features['evening'] = all_features['hour'].apply(lambda x: 1
if x >= 18 else 0)
 wkm = {
'Monday': 0,
'Tuesday': 1,
'Wednesday': 2,
'Thursday': 3,
'Friday': 4,
'Saturday': 5,
'Sunday': 6
}
all_features['DayOfWeek'] = all_features['DayOfWeek'].apply(lambda x: wkm[x])
 all_features['weekend'] = all_features['DayOfWeek'].apply(
 lambda x: 1 if x == 4 or x == 5 else 0)
 OneHot_features = pd.get_dummies(all_features['PdDistrict'])
 all_features['block'] = all_features['Address'].apply(
 lambda x: 1 if 'block' in x.lower() else 0)
 PCA_features = all_features[['X', 'Y']].values
Standard_features = all_features[['DayOfWeek', 'year', 'month', 'day',
'hour']].values
OneHot_features = pd.concat([
OneHot_features, all_features[['new_year', 'evening', 'weekend', 'block']]
],
axis=1 ).values
scaler = StandardScaler()
scaler.fit(Standard_features)
 Standard_features = scaler.transform(Standard_features)
 pca = PCA(n_components=2)
 pca.fit(PCA_features)
 PCA_features = pca.transform(PCA_features)
 all_features = np.concatenate(
(PCA_features, Standard_features, OneHot_features), axis=1)
 train_features = all_features[:num_train]
num_inputs = train_features.shape[1]
test_features = all_features[num_train:]<create_dataframe>
test_preds = np.zeros(( test_features_df.shape[0], y.shape[1]))
losses = []
SEED = [0]
model = Model(
 num_features=X.shape[1],
num_targets=y.shape[1],
hidden_size=1500, 
)
 skf = MultilabelStratifiedKFold(n_splits = 5)
 for i in SEED:
seed_everything(i)
 for fold,(train_index, valid_index)in enumerate(skf.split(X, y)) :
gc.collect()
torch.cuda.empty_cache()
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
valid_dataset = MoaDataset(X_valid, y_valid)
 train_dataset = MoaDataset(X_train, y_train)
 dls = DataLoaders.from_dsets(train_dataset, valid_dataset, bs=128)
 dls = dls.cuda()
model = Model(
 num_features=X.shape[1],
num_targets=y.shape[1],
hidden_size=2000, 
)
 cbs=SaveModelCallback()
model.to('cuda')
 loss_func = SmoothBCEwLogits(smoothing=0.001)
 learner = Learner(dls, model, loss_func = loss_func, opt_func=CustomAdamW)
 learner.fit_one_cycle(100, cbs=[EarlyStoppingCallback(monitor='valid_loss', patience=5),cbs],pct_start=0.1, div_factor=1000,
max_lr=1e-3)
 log_loss = np.array(learner.recorder.values)[:,1].min()
learner.load('./model')
 learner.predict(torch.tensor(X_test.values, dtype=torch.float))
preds = learner.pred
test_preds += preds.sigmoid().detach().cpu().numpy()
losses.append(log_loss)
 print(np.array(losses ).sum() /7)
 test_preds[control_mask] = 0
data_train = lgb.Dataset(train_features, label = train_labels )<load_from_csv>
sample_sub_df.iloc[:,1:] = test_preds/5
sample_sub_df.to_csv('submission.csv', index=False )