Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<set_options>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 PATH = '.. /input/abstraction-and-reasoning-challenge'
PATH_TESTS = PATH + '/test'
EPOCHS = 100
BATCH = 4
LR = 0.0001
SEED = 1234
random_seed(SEED)
 TOKENS = ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '[
with open('vocab.txt', 'w')as fp:
for t in TOKENS:
fp.write(t + '
')
 def train(name, epochs=EPOCHS):
ds_train = TaskDataset(PATH_TESTS, name, mode='train')
 ld_train = torch.utils.data.DataLoader(ds_train, batch_size=BATCH, num_workers=4, shuffle=True)
 w = ds_train.task_width
h = ds_train.task_height
model = Network(w, h)
 model = model.to(DEVICE)
 criterion = nn.CrossEntropyLoss()
optimizer = op.Adam(model.parameters() , lr=LR)
 metrics = CrossEntropyMetrics()
summary_train = Summary()
for epoch in range(epochs):
model.train()
metrics.begin()
for step,(id, x, y)in enumerate(ld_train):
optimizer.zero_grad()
x0 = x[0].to(DEVICE)
 x1 = x[1].to(DEVICE)
 x2 = x[2].to(DEVICE)
 dy = y[0].to(DEVICE)
 dz = model(( x0, x1, x2))
loss = criterion(dz[:,:,1:], dy[:,1:])
 metrics.update(loss, dz, dy)
 loss.backward()
optimizer.step()
accuracy = metrics.commit(epoch)
 summary_train.register(epoch, loss.item() , accuracy, metrics.values)
 print(summary_train)
 return model
def predict(model, name):
ds_tests = TaskDataset(PATH_TESTS, name, mode='tests')
 ld_tests = torch.utils.data.DataLoader(ds_tests, batch_size=1, shuffle=False)
 w = ds_tests.task_width
h = ds_tests.task_height
preds = []
model.eval()
with torch.no_grad() :
for step,(id, x, y)in enumerate(ld_tests):
x0 = x[0].to(DEVICE)
 x1 = x[1].to(DEVICE)
 x2 = x[2].to(DEVICE)
 dz = model(( x0, x1, x2))
z = dz.detach().cpu().numpy()
z = z[0]
az = z.argmax(0)
 w, h = Task.pred_size(az)
 w = max(1, w)
 h = max(1, h)
 pred = Task.as_array(az, w, h)
 pred = Task.as_list(pred)
 flat = flattener(pred)
 preds.append(flat)
 return preds
submission = pd.read_csv(PATH + '/sample_submission.csv', index_col='output_id')
 STEPS = -1
step = 0
hash = set()
for output_id in submission.index:
task_id = output_id.split('_')[0]
if task_id in hash:
continue
hash.add(task_id)
 name = str(task_id + '.json')
 model = train(name, EPOCHS)
 preds = predict(model, name)
 for n in range(len(preds)) :
pred = preds[n] + ' ' + preds[n] + ' ' + preds[n]
submission.loc[F'{task_id}_{n}', 'output'] = pred
step += 1
if step == STEPS: break
submission.to_csv('submission.csv')
<categorify>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
df = pd.read_csv('submission.csv')
 df.head()
tokenizer = BertTokenizer('vocab.txt')
 def tokenize(sequ, max_length):
sequ_dict = tokenizer.encode_plus(sequ, add_special_tokens=False, max_length=max_length, pad_to_max_length=True)
 return np.array(sequ_dict['input_ids'], dtype=np.int64)
 for values in df.values:
name = values[0]
name = name[:-2] + '.json'
print('TRAIN', name)
 task = Task(PATH_TESTS, name)
 for inp, out in task.get_train() :
task_width, task_height, _ = task.max_size()
max_length = 2 +(task_width + 1)* task_height
x = tokenize(inp, max_length)
 y = tokenize(out, max_length)
 wx, hx = Task.pred_size(x)
 arr_x = Task.as_array(x, wx, hx)
 wy, hy = Task.pred_size(y)
 arr_y = Task.as_array(y, wy, hy)
 plot_task_array([arr_x, arr_y])
 ix = 0
print('PREDS', name)
 for inp in task.get_tests() :
try:
x = tokenize(inp, max_length)
 wx, hx = Task.pred_size(x)
 arr_x = Task.as_array(x, wx, hx)
 parts = values[1].split(' ')[0].split('|')
 parts = parts[1:-1]
h = len(parts)
 w = len(parts[0])
 a = np.zeros(( h, w), dtype='int')
 for y in range(h):
for x in range(w):
a[y][x] = parts[y][x]
plot_task_array([arr_x, a])
 except Exception as ex:
print(ex)
 print()<set_options>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
rc('animation', html='jshtml')
 device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<define_variables>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge')
 train_path = data_path / 'training'
eval_path = data_path / 'evaluation'
test_path = data_path / 'test'
train_tasks = { task.stem: json.load(task.open())for task in train_path.iterdir() }
eval_tasks = { task.stem: json.load(task.open())for task in eval_path.iterdir() }
test_tasks = { task.stem: json.load(task.open())for task in test_path.iterdir() }<choose_model_class>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )
HIDDEN_SIZE = 128
MAX_STEPS = 10
THRESHOLD = 0.99
REMAINDERS_PEN = 0.0
torch.manual_seed(42)
 np.random.seed(42)
 class CAModel(nn.Module):
def __init__(self, num_states):
super(CAModel, self ).__init__()
self.embedding = nn.Sequential(
 nn.Conv2d(num_states, HIDDEN_SIZE, kernel_size=1),
nn.InstanceNorm2d(HIDDEN_SIZE), 
)
 self.transition = nn.Sequential(
 nn.InstanceNorm2d(HIDDEN_SIZE),
nn.Conv2d(HIDDEN_SIZE, HIDDEN_SIZE, kernel_size=3, padding=1),
nn.ReLU() ,
nn.Conv2d(HIDDEN_SIZE, HIDDEN_SIZE, kernel_size=1, padding=0), 
)
 self.projection_out = nn.Conv2d(HIDDEN_SIZE, num_states, kernel_size=1)
 self.projection_halt = nn.Conv2d(HIDDEN_SIZE, 1, kernel_size=1)
 def forward(self, x, max_steps=None):
x = self.embedding(x)
 halting_probability = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 remainders = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 n_updates = torch.zeros([1, 1, x.shape[2], x.shape[3]], device=x.device)
 max_steps = max_steps or MAX_STEPS
for i in range(max_steps):
p = torch.sigmoid(self.projection_halt(x)- 1)
 still_running =(halting_probability <= THRESHOLD ).to(torch.float)
 new_halted =(( halting_probability + p * still_running)> THRESHOLD ).to(torch.float)* still_running
still_running =(( halting_probability + p * still_running)<= THRESHOLD ).to(torch.float)* still_running
halting_probability += p * still_running
remainders += new_halted *(1 - halting_probability)
 halting_probability += new_halted * remainders
n_updates += still_running + new_halted
update_weights = p * still_running + new_halted * remainders
transformed_state = self.transition(x)
 x = transformed_state * update_weights + x *(1 - update_weights)
 if still_running.sum() == 0:
break
x = self.projection_out(x)
 self._remainders = remainders
self._n_updates = n_updates
return x<train_on_grid>
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )
def solve_task(task, max_steps=10):
model = CAModel(10 ).to(device)
 num_epochs = 100
criterion = nn.CrossEntropyLoss()
losses = np.zeros(num_epochs)
 n_updates = np.zeros(num_epochs)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.001)
 for e in range(num_epochs):
optimizer.zero_grad()
loss = 0.0
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 y = torch.tensor(sample["output"] ).long().unsqueeze(0 ).to(device)
 y_pred = model(x)
 loss += criterion(y_pred, y)+(model._remainders * REMAINDERS_PEN ).mean(0 ).sum()
n_updates[e] += model._n_updates.detach().cpu().mean().numpy() / len(task)
 loss.backward()
optimizer.step()
losses[e] = loss.item()
return model, losses, n_updates
@torch.no_grad()
def predict(model, task):
predictions = []
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 pred = model(x ).argmax(1 ).squeeze().cpu().numpy()
predictions.append(pred)
 return predictions
task = train_tasks["db3e9e38"]["train"]
model, losses, n_updates = solve_task(task )<predict_on_test>
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 cols_numeric = [feat for feat in train.columns if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
print('Train shape:', train.shape)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], test.loc[:, cols_numeric]], axis=1)
 print('Test shape:', test.shape)
predictions = predict(model, task)
 for i in range(len(task)) :
plot_sample(task[i], predictions[i] )<predict_on_test>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
test = train_tasks["db3e9e38"]["test"]
predictions = predict(model, test)
 for i in range(len(test)) :
plot_sample(test[i], predictions[i] )<predict_on_test>
def evaluate(tasks, is_test=False):
result = []
predictions = {}
for idx, task in tqdm(tasks.items()):
if input_output_shape_is_same(task):
model, _, _ = solve_task(task["train"])
 pred = predict(model, task["test"])
 if not is_test:
score = calk_score(task["test"], pred)
 else:
score = [0] * len(task["test"])
 else:
pred = [el["input"] for el in task["test"]]
score = [0] * len(task["test"])
 predictions[idx] = pred
result.append(score)
 return result, predictions<compute_test_metric>
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 PCA Test:')
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
train_result, train_predictions = evaluate(train_tasks)
 train_solved = [any(score)for score in train_result]
total = sum([len(score)for score in train_result])
 print(f"solved : {sum(train_solved)} from {total}({sum(train_solved)/total})" )<compute_test_metric>
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"] )
eval_result, eval_predictions = evaluate(eval_tasks)
 eval_solved = [any(score)for score in eval_result]
total_eval = sum([len(score)for score in eval_result])
 print(f"Eval: solved : {sum(eval_solved)} from {total_eval}({sum(eval_solved)/total})" )<compute_test_metric>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1 )
test_result, test_predictions = evaluate(test_tasks, is_test=True )<load_from_csv>
features_to_drop = ["sig_id", "cp_type"]
train.drop(features_to_drop, axis = 1, inplace = True)
 test.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = train
test_df = test
train_df.reset_index(drop = True, inplace = True)
 test_df.reset_index(drop = True, inplace = True )
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )<categorify>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
def flattener(pred):
str_pred = str([list(row)for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<load_from_disk>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )
for output_id in submission.index:
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = str(test_path / str(task_id + '.json'))
with open(f, 'r')as read_file:
task = json.load(read_file)
 data = test_predictions[task_id][pair_id]
pred_1 = flattener(data)
 data = [[5 if i==0 else i for i in j] for j in data]
pred_2 = flattener(data)
 data = [[0 for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv')
 submission.head()<define_variables>
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(len(training_tasks), len(evaluation_tasks), len(test_tasks)) 
<load_from_disk>
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 10
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
SEED = [42]
for s in SEED:
tabnet_params['seed'] = s
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s, shuffle = True)
 for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_nb+1)
 saved_filepath = model.save_model(saving_path_name)
 loaded_model = TabNetRegressor()
loaded_model.load_model(saved_filepath)
 preds_val = loaded_model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
def get_data(task_filename):
with open(task_filename, 'r')as f:
task = json.load(f)
 return task
num2color = ["black", "blue", "red", "green", "yellow", "gray", "magenta", "orange", "sky", "brown"]
color2num = {c: n for n, c in enumerate(num2color)}<categorify>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
def task_train000(x):
x_upsampled = x.repeat(3, axis=0 ).repeat(3, axis=1)
 x_tiled = np.tile(x,(3, 3))
y = x_upsampled & x_tiled
return y
task = get_data(str(training_path / training_tasks[0]))
check(task, task_train000 )<prepare_output>
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()