Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<load_from_csv>
warnings.filterwarnings('ignore' )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
<categorify>
%%time
train_df = pd.read_csv(os.path.join(PATH,"train.csv"))
test_df = pd.read_csv(os.path.join(PATH,"test.csv"))
def preprocess(df):

 df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 del df['sig_id']
return df<drop_column>
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<filter>
%%time
missing_data(train_df )
train_targets = train_targets.loc[train['cp_type']==0].reset_index(drop=True)
 train = train.loc[train['cp_type']==0].reset_index(drop=True )<normalization>
%%time
missing_data(test_df )
scaler = MinMaxScaler()
scaler.fit(train)
 train = scaler.transform(train)
 test = scaler.transform(test )<data_type_conversions>
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))
train_targets = train_targets.to_numpy()<categorify>
%%time
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
return {
'input': torch.tensor(self.features[idx, :], dtype=torch.float),
'target': torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
return {
'input': torch.tensor(self.features[idx, :], dtype=torch.float)
 }<normalization>
%%time
features = [c for c in train_df.columns if c not in ['id', 'target']]
for df in [test_df, train_df]:
df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1 )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
device = 'cuda' if torch.cuda.is_available() else 'cpu'
self.weight = torch.tensor([0.5] ).to(device)
 def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.prelu(self.dense1(x), self.weight)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.prelu(self.dense2(x), self.weight)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<set_options>
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )
def set_seed(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed )<create_dataframe>
def apply_noise(data, noise_level):
idxt = data[['id', 'target']]
features = data.columns.values[2:]
appended_data = []
for feature in features:
signal = data[feature]
noise_factor =(np.abs(signal)).mean() * noise_level
noise = np.random.normal(0, noise_level, signal.shape)
 jittered = signal + noise
appended_data.append(pd.DataFrame(jittered))
appended_data = pd.concat(appended_data, axis=1)
 data_jittered = pd.concat([idxt, pd.DataFrame(appended_data)], axis=1)
 return data_jittered
def get_dataloaders(num_workers, batch_size, x_train, y_train, x_valid, y_valid):

 train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
"train": train_loader,
"valid": valid_loader
}
return loader<load_pretrained>
noise_train_df = []
for i in tqdm_notebook(range(0,2)) :
t = apply_noise(train_df, noise_level = i * 0.025)
 noise_train_df.append(t)
 noise_train_df = pd.concat(noise_train_df, axis = 0 )
def get_testloaders(num_workers, batch_size, x_test):

 test_dataset = TestDataset(x_test)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
'test': test_loader
}
return loader<train_model>
features = [c for c in train_df.columns if c not in ['id', 'target']]
target = train_df['target']
def train_model(n_epochs, loaders, model, optimizer, criterion, device, save_path):

 scheduler = StepLR(optimizer, step_size=2, gamma=0.96)
 valid_loss_min = np.Inf
for epoch in range(1, n_epochs + 1):
scheduler.step()
train_loss = 0.0
valid_loss = 0.0
model.train()
for data in loaders['train']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 optimizer.zero_grad()
output = model(data_input)
 loss = criterion(output, data_target)
 loss.backward()
optimizer.step()
train_loss +=(loss.item() / len(loaders['train']))
model.eval()
for data in loaders['valid']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 output = model(data_input)
 loss = criterion(output, data_target)
 valid_loss +=(loss.item() / len(loaders['valid']))
if valid_loss < valid_loss_min:
print(f'Epoch: \t{epoch}\tValidation loss decreased({valid_loss_min} -> {valid_loss} ).Saving the model...')
 torch.save(model.state_dict() , save_path)
 valid_loss_min = valid_loss
clear_output(wait=True)
 return model<split>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.83,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.81,
'learning_rate': 0.005,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 90,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 11,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
def run_training(seed, kfold, batch_size, epochs, learning_rate, weight_decay):
set_seed(seed)
 for fold,(train_idx, valid_idx)in enumerate(kfold.split(X=train, y=train_targets)) :
x_train, x_valid = train[train_idx], train[valid_idx]
y_train, y_valid = train_targets[train_idx], train_targets[valid_idx]
dataloaders = get_dataloaders(0, batch_size, x_train, y_train, x_valid, y_valid)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024 ).to(device)
 criterion_moa = nn.BCEWithLogitsLoss()
optimizer_moa = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=weight_decay)
 train_model(epochs, dataloaders, model, optimizer_moa, criterion_moa, device, f'/models/model_seed_{seed}_fold_{fold}.pt' )<define_variables>
folds = StratifiedKFold(n_splits=9, shuffle=True, random_state=4422)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 100000
clf = lgb.train(param, trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=400,
early_stopping_rounds = 400)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
<train_model><EOS> 
sub_df = pd.DataFrame({"id":test_df["id"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )