Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<predict_on_test>
warnings.filterwarnings('ignore')
 np.random.seed(27 )
def inference(loaders, model, device):

 model.eval()
preds = []
for data in loaders['test']:
data_input = data['input'].to(device)
 with torch.no_grad() :
output = model(data_input)
 pred = output.sigmoid().detach().cpu().numpy()
preds.append(pred)
 return np.concatenate(preds )<define_search_space>
%matplotlib inline
plt.rcParams['figure.figsize'] = [20.0, 7.0]
plt.rcParams.update({'font.size': 22,})
 sns.set_palette('viridis')
 sns.set_style('white')
 sns.set_context('talk', font_scale=0.8 )
def run_inferencing(seed):
set_seed(seed)
 preds = np.zeros(( len(test), 206))
for i in range(0, FOLDS):
dataloaders = get_testloaders(0, 128, test)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024)
 model.load_state_dict(torch.load(f'/models/model_seed_{seed}_fold_{i}.pt'))
model.to(device)
 pred = inference(dataloaders, model, device)
 preds += pred
preds = preds / FOLDS
return preds 
<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('Train Shape: ', train.shape)
 print('Test Shape: ', test.shape)
 train.head()
preds = np.zeros(( len(test), 206))
for seed in range(35, 40):
preds += run_inferencing(seed)
 preds = preds / 5<save_to_csv>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
sample_submission[targets] = preds
sample_submission.loc[test_features['cp_type']=='ctl_vehicle', targets] = 0
sample_submission.to_csv('submission.csv', index=False )<set_options>
ridge = linear_model.Ridge()
lasso = linear_model.Lasso()
elastic = linear_model.ElasticNet()
lasso_lars = linear_model.LassoLars()
bayesian_ridge = linear_model.BayesianRidge()
logistic = linear_model.LogisticRegression(solver='liblinear')
 sgd = linear_model.SGDClassifier()
warnings.filterwarnings('ignore')
 def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<normalization>
models = [ridge, lasso, elastic, lasso_lars, bayesian_ridge, logistic, sgd]
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
count = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 scheduler.step()
return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<load_from_csv>
def get_cv_scores(model):
scores = cross_val_score(model, X_train, y_train, cv=5, scoring='roc_auc')
 print('CV Mean: ', np.mean(scores))
print('STD: ', np.std(scores))
print('
' )
data_loc = ".. /input/lish-moa/"
train_features = pd.read_csv(data_loc + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_loc + 'train_targets_scored.csv')
 test_features = pd.read_csv(data_loc + 'test_features.csv')
 target_cols = [x for x in train_targets_scored.columns.to_list() if x != 'sig_id']
print(train_features.shape, test_features.shape )<define_variables>
for model in models:
print(model)
 get_cv_scores(model )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
penalty = ['l1', 'l2']
C = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]
class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]
solver = ['liblinear', 'saga']
param_grid = dict(penalty=penalty,
C=C,
class_weight=class_weight,
solver=solver)
 grid = GridSearchCV(estimator=logistic, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1)
 grid_result = grid.fit(X_train, y_train)
 print('Best Score: ', grid_result.best_score_)
 print('Best Params: ', grid_result.best_params_ )
feature_to_scale = ["cp_time"] + GENES + CELLS
print(len(feature_to_scale))
data_scaler = MinMaxScaler()
data_entire[feature_to_scale ] = data_scaler.fit_transform(data_entire[feature_to_scale ] )<categorify>
logistic = linear_model.LogisticRegression(C=1, class_weight={1:0.6, 0:0.4}, penalty='l1', solver='liblinear')
 get_cv_scores(logistic )
print(data_entire.shape)
 cols_ohe = ['cp_type', 'cp_dose']
data_entire = ohe(data_entire, cols_ohe)
<split>
predictions = logistic.fit(X_train, y_train ).predict_proba(X_test)
data_train = data_entire.iloc[0:23814, :].copy(deep=True)
 data_test = data_entire.iloc[23814:, :].copy(deep=True )<load_pretrained>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.head()
tmp_data_dir = "/kaggle/working/tmp_data/"
try:
os.makedirs(tmp_data_dir)
 except FileExistsError:
pass
data_train.to_pickle(tmp_data_dir + "data_train.pkl")
 data_test.to_pickle(tmp_data_dir + "data_test.pkl" )<load_pretrained>
loss = ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron']
penalty = ['l1', 'l2', 'elasticnet']
alpha = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]
learning_rate = ['constant', 'optimal', 'invscaling', 'adaptive']
class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]
eta0 = [1, 10, 100]
param_distributions = dict(loss=loss,
penalty=penalty,
alpha=alpha,
learning_rate=learning_rate,
class_weight=class_weight,
eta0=eta0)
 random = RandomizedSearchCV(estimator=sgd, param_distributions=param_distributions, scoring='roc_auc', verbose=1, n_jobs=-1, n_iter=1000)
 random_result = random.fit(X_train, y_train)
 print('Best Score: ', random_result.best_score_)
 print('Best Params: ', random_result.best_params_ )
seed_everything(seed=1903)
 tmp_data_dir = "/kaggle/working/tmp_data/"
data_train_x = pd.read_pickle(tmp_data_dir + "data_train.pkl")
 data_test_x = pd.read_pickle(tmp_data_dir + "data_test.pkl")
 target_cols = [x for x in train_targets_scored.columns.to_list() if x != 'sig_id']
feature_cols = [c for c in data_train_x.columns.to_list() if c != 'sig_id']<merge>
sgd = linear_model.SGDClassifier(alpha=0.1,
class_weight={1:0.7, 0:0.3},
eta0=100,
learning_rate='optimal',
loss='log',
penalty='elasticnet')
 get_cv_scores(sgd )
train_entire = data_train_x.merge(train_targets_scored, on="sig_id")
 NFOLDS = 5
seed = 1903
mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=data_train_x, y=train_targets_scored[target_cols])) :
train_entire.loc[v_idx, 'kfold'] = int(f)
 train_entire['kfold'] = train_entire['kfold'].astype(int )<load_pretrained>
predictions = sgd.fit(X_train, y_train ).predict_proba(X_test)
train_entire.to_pickle(tmp_data_dir + "data_train_process2_5fold_1903.pkl" )<prepare_x_and_y>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()