Kaggle competition: Mechanisms of Action (MoA) Prediction
parameters = {'max_depth':np.arange(3,10,2)}
dt = DecisionTreeClassifier()
dt_grid = GridSearchCV(dt,param_grid=parameters, n_jobs=-1)
 dt_grid_results = perform_model(dt_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(dt_grid_results['model'] )<train_on_grid>
train = pd.read_csv(file_path("train_features.csv")).sort_values(by='sig_id')
 targets = pd.read_csv(file_path("train_targets_scored.csv")).sort_values(by='sig_id')
 test = pd.read_csv(file_path("test_features.csv"))
submission = test[['sig_id']].assign(**targets.iloc[:, 1:].mean() )
n_estimators = [10, 100, 500, 1000, 2000]
max_depth = [5, 10, 20]
parameters = dict(n_estimators=n_estimators, max_depth=max_depth)
 rf = RandomForestClassifier(random_state=42)
 rf_grid = GridSearchCV(rf,param_grid=parameters, n_jobs=-1)
 rf_grid_results = perform_model(rf_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(rf_grid_results['model'] )<train_model>
mask = test['cp_type'] != 'ctl_vehicle'
submission.iloc[~mask, 1:] = 0
model_rf_final = RandomForestClassifier(max_depth= 5, n_estimators= 500)
 model_rf_final.fit(X_train, y_train )<predict_on_test>
def basic_preprocess(X, y=None, pca=None):
mask = np.where(X['cp_type'] == 'ctl_vehicle')[0]
X.drop(mask, inplace=True)
 if y is not None:
y.drop(mask, inplace=True)
 y.drop(columns='sig_id', inplace=True)
 X.drop(columns=['cp_type', 'sig_id'], inplace=True)
 X['cp_dose'] =(( X['cp_dose'] == 'D2' ).astype(np.int)- 0.5)* 2
X['cp_time1'] =(( X['cp_time'] == 24 ).astype(np.int)- 0.5)* 2
X['cp_time2'] =(( X['cp_time'] == 48 ).astype(np.int)- 0.5)* 2
X.drop(columns='cp_time', inplace=True)
 if y is not None:
return X, y
return X
def preprocess(train, targets, test, pca=None, scaler=None, label_smoothing=0):
train, targets = basic_preprocess(train, targets)
 test = basic_preprocess(test)
 if pca:
mask = train.columns.str.startswith('g-')
 train.loc[:, mask] = pca.fit_transform(train.loc[:, mask])
 test.loc[:, mask] = pca.transform(test.loc[:, mask])
 mask = train.columns.str.startswith('c-')
 train.loc[:, mask] = pca.fit_transform(train.loc[:, mask])
 test.loc[:, mask] = pca.transform(test.loc[:, mask])
 if scaler:
mask = train.columns.str.startswith('g-')
 train.loc[:, mask] = scaler.fit_transform(train.loc[:, mask])
 test.loc[:, mask] = scaler.transform(test.loc[:, mask])
 mask = train.columns.str.startswith('c-')
 train.loc[:, mask] = scaler.fit_transform(train.loc[:, mask])
 test.loc[:, mask] = scaler.transform(test.loc[:, mask])
 if label_smoothing:
targets.clip(label_smoothing, 1 - label_smoothing, inplace=True)
 return train, targets, test
label_smoothing = 5e-4
train, targets, test = preprocess(train, targets, test, label_smoothing=0 )
test_pred = pd.Series(model_rf_final.predict(test), name = "Survived")
 test_pred_final = pd.DataFrame(test_pred )<load_from_csv>
X = torch.tensor(train.values, dtype=torch.float)
 y = torch.tensor(targets.values, dtype=torch.float)
 X_t = torch.tensor(test.values, dtype=torch.float)
 dataset = TensorDataset(X, y)
 test_dataset = TensorDataset(X_t, torch.zeros(X_t.shape[0], y.shape[1], dtype=y.dtype))
df_gender_submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
<save_to_csv>
class CrafterModel(nn.Module):
def __init__(self, input_size, output_size, label_clip=label_smoothing):
super().__init__()
self.model = nn.Sequential()
if label_smoothing:
self.clip_min = torch.log(torch.tensor(label_clip)) - torch.log(torch.tensor(1 - label_clip))
else:
self.clip_min = -float("inf")
 hidden_sizes = [2048, 2048, output_size]
dropouts = [0.2, 0.4, 0.4]
assert len(hidden_sizes)== len(dropouts)
 for i,(hidden_size, dropout)in enumerate(zip(hidden_sizes, dropouts), start=1):
self.model.add_module(f"batch_norm{i}", nn.BatchNorm1d(input_size))
self.model.add_module(f"dropout{i}", nn.Dropout(dropout))
linear_layer = torch.nn.utils.weight_norm(nn.Linear(input_size, hidden_size))
self.model.add_module(f"linear{i}", linear_layer)
 if i != len(hidden_sizes):
self.model.add_module(f"activation{i}", nn.ELU())
 input_size = hidden_size
def forward(self, input):
input = self.model.forward(input)
 input = torch.clamp(input, self.clip_min, -self.clip_min)
 return input
class CustomRunner(dl.SupervisedRunner):
loss = nn.BCEWithLogitsLoss(reduction='mean')
 def _handle_batch(self, batch):
y_pred = self.model(batch['features'])
 loss = CustomRunner.loss(y_pred, batch['targets'])
 self.batch_metrics.update({"loss": loss})
 class EpochMetricSaverCallback(dl.Callback):
def __init__(self, metrics=None):
super().__init__(dl.CallbackOrder.Logging)
 self.metrics_log = defaultdict(lambda: [])
 self.metrics = metrics
def on_epoch_end(self, runner):
metrics = self.metrics
if metrics is None:
metrics = runner.epoch_metrics
for x in metrics:
self.metrics_log[x].append(runner.epoch_metrics[x] )
submission = pd.DataFrame({
"PassengerId": testo["PassengerId"],
"Survived": test_pred_final['Survived']
})
 submission.to_csv('Titanic Submission.csv', index = False)
 print('Done' )<load_from_csv>
def train_model(train_dataset, valid_dataset, num_epochs=15):
loaders = {
"train": DataLoader(train_dataset, batch_size=128, shuffle=True),
"valid": DataLoader(valid_dataset, batch_size=1024, shuffle=False)
 }
model = CrafterModel(X.shape[1], y.shape[1])
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=1e-5)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 logdir = "./logs"
metric_save = EpochMetricSaverCallback()
runner = CustomRunner()
runner.train(
 model=model,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
num_epochs=num_epochs,
logdir=logdir,
callbacks=[
dl.EarlyStoppingCallback(5),
metric_save
],
verbose=True,
load_best_on_end=True 
)
 return model, metric_save.metrics_log
def predict(model, dataset):
runner = dl.SupervisedRunner()
results = runner.predict_loader(
 model=model,
loader=DataLoader(dataset, batch_size=1024)
)
 total_results = []
for x in results:
total_results.append(torch.sigmoid(x['logits'] ).to('cpu'))
total_results = torch.cat(total_results)
 return total_results
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
models = []
predictions = []
metric_logs = []
n_splits = 5
for train_idx, valid_idx in tqdm(KFold(n_splits=n_splits, shuffle=True, random_state=RANDOM_SEED ).split(dataset), total=n_splits):
train_dataset = Subset(dataset, train_idx)
 valid_dataset = Subset(dataset, valid_idx)
 model, metric_log = train_model(train_dataset, valid_dataset)
 models.append(model.to('cpu'))
metric_logs.append(metric_log)
 predictions.append(predict(model, test_dataset))
predictions = torch.stack(predictions )
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
print(test_data.isnull().sum() )<split>
total_results = predictions.mean(dim=0)
 total_results.clamp_(label_smoothing, 1 - label_smoothing )
y = train_data["Survived"]
features = ["Pclass", "Sex", "SibSp", "Parch", "Fare", "Age"]
X = pd.get_dummies(train_data[features])
 X_unknown = pd.get_dummies(test_data[features])
 my_imputer = SimpleImputer()
X = my_imputer.fit_transform(X)
 X_unknown = my_imputer.fit_transform(X_unknown)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=10 )<train_model>
submission.iloc[np.where(mask)[0], 1:] = total_results
<train_model><EOS> 
submission.to_csv("submission.csv", index=None, float_format='%.7f' )