Kaggle competition: Mechanisms of Action (MoA) Prediction
labels = df.pop('target')
 data = df.drop('ID_code', axis=1 )<load_from_csv>
warnings.filterwarnings('ignore')
test_data = pd.read_csv(".. /input/test.csv")
 test_data.head()<drop_column>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
test_data = test_data.drop('ID_code', axis=1)
 test_data.head()<choose_model_class>
scored = train_targets_scored.copy()
drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 targets = scored.columns[1:]
scored = scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
SEED = 42
FOLDS = 7
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['kfold'] = scored.drug_id.map(dct1)
 scored.loc[scored.kfold.isna() ,'kfold'] = scored.loc[scored.kfold.isna() ,'sig_id'].map(dct2)
 scored.kfold = scored.kfold.astype('int8')
 train_targets_scored = scored.copy()
del scored
n_splits = 10
kf = KFold(n_splits=n_splits )<train_model>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
warnings.filterwarnings('ignore')
 param = {'objective': 'binary', 'metric': 'auc', 'learning_rate': 0.01, 'num_rounds': 6000, 'verbose': 1}
test_pred = np.zeros(len(test_data))
for fold,(train_indx, val_indx)in enumerate(kf.split(labels)) :
print("Fold {}".format(fold+1))
train_set = lgb.Dataset(data.iloc[train_indx], label=labels.iloc[train_indx])
 val_set = lgb.Dataset(data.iloc[val_indx], label=labels.iloc[val_indx])
 model = lgb.train(param, train_set, valid_sets=val_set, verbose_eval=500)
 test_pred += model.predict(test_data)/n_splits<save_to_csv>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=125,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
save = pd.read_csv('.. /input/sample_submission.csv')
 save['target'] = test_pred
save.to_csv('LGB_kfold.csv', index=False)
 save.head()<import_modules>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
import numpy as np
import pandas as pd
import numba
from sympy import isprime, primerange
from math import sqrt
from sklearn.neighbors import KDTree
from tqdm import tqdm
from itertools import combinations<load_from_csv>
n_comp = 100
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
cities = pd.read_csv('.. /input/traveling-santa-2018-prime-paths/cities.csv', index_col=['CityId'])
 XY = np.stack(( cities.X.astype(np.float32), cities.Y.astype(np.float32)) , axis=1)
 is_not_prime = np.array([0 if isprime(i)else 1 for i in cities.index], dtype=np.int32 )<compute_test_metric>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 train_features.shape
@numba.jit('f8(i8, i8, i8)', nopython=True, parallel=False)
 def cities_distance(offset, id_from, id_to):
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 if offset % 10 == 9 and is_not_prime[id_from]:
return 1.1 * distance
return distance
@numba.jit('f8(i4, i8[:])', nopython=True, parallel=False)
 def score_chunk(offset, chunk):
pure_distance, penalty = 0.0, 0.0
penalty_modulo = 9 - offset % 10
for path_index in numba.prange(chunk.shape[0] - 1):
id_from, id_to = chunk[path_index], chunk[path_index+1]
xy_from, xy_to = XY[id_from], XY[id_to]
dx, dy = xy_from[0] - xy_to[0], xy_from[1] - xy_to[1]
distance = sqrt(dx * dx + dy * dy)
 pure_distance += distance
if path_index % 10 == penalty_modulo and is_not_prime[id_from]:
penalty += distance
return pure_distance + 0.1 * penalty
@numba.jit('f8(i8[:])', nopython=True, parallel=False)
 def score_path(path):
return score_chunk(0, path )<choose_model_class>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
del target['kfold']
del target['drug_id']
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
kdt = KDTree(XY )<load_from_csv>
folds = train.copy()
del train['drug_id']
del train['kfold']
del folds['drug_id']
folds.head()
path = np.array(pd.read_csv('.. /input/santa-2018/submission_1515572.48.csv' ).Path)
 initial_score = score_path(path )<compute_test_metric>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
path_index = np.argsort(path[:-1])
 total_score = initial_score
print(f'Total score is {total_score:.2f}.')
 for _ in range(5):
for step,(id1, id2)in enumerate(pairs[:10**6], 1):
if step % 10**6 == 0:
new_total_score = score_path(path)
 print(f'Score: {new_total_score:.2f}; improvement over last 10^6 steps: {total_score - new_total_score:.2f}; total improvement: {initial_score - new_total_score:.2f}.')
 total_score = new_total_score
i, j = path_index[id1], path_index[id2]
i, j = min(i, j), max(i, j)
 chunk, reversed_chunk = path[i-1:j+2], np.concatenate([path[i-1:i], path[j:i-1:-1], path[j+1:j+2]])
 chunk_score, reversed_chunk_score = score_chunk(i-1, chunk), score_chunk(i-1, reversed_chunk)
 if j - i > 2:
chunk_abc = np.concatenate([path[i-1:i+1], path[j:i:-1], path[j+1:j+2]])
 chunk_acb = np.concatenate([path[i-1:i], path[j:j+1], path[i:j], path[j+1:j+2]])
 chunk_abcb = np.concatenate([path[i-1:i+1], path[j:j+1], path[i+1:j], path[j+1:j+2]])
 abc_score, acb_score, abcb_score = map(lambda chunk: score_chunk(i-1, chunk), [chunk_abc, chunk_acb, chunk_abcb])
 for chunk, score, name in zip(( chunk_abc, chunk_acb, chunk_abcb),(abc_score, acb_score, abcb_score),('abc', 'acb', 'abcb')) :
if score < chunk_score:
path[i-1:j+2] = chunk
path_index = np.argsort(path[:-1])
 chunk_score = score
if reversed_chunk_score < chunk_score:
path[i-1:j+2] = reversed_chunk
path_index = np.argsort(path[:-1] )<compute_test_metric>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
print(f'Total improvement is {initial_score - total_score:.2f}.' )<save_to_csv>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )<compute_test_metric>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.26)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.26)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def recalibrate_layer(self, layer):
if(torch.isnan(layer.weight_v ).sum() > 0):
print('recalibrate layer.weight_v')
 layer.weight_v = torch.nn.Parameter(torch.where(torch.isnan(layer.weight_v), torch.zeros_like(layer.weight_v), layer.weight_v))
layer.weight_v = torch.nn.Parameter(layer.weight_v + 1e-7)
 if(torch.isnan(layer.weight ).sum() > 0):
print('recalibrate layer.weight')
 layer.weight = torch.where(torch.isnan(layer.weight), torch.zeros_like(layer.weight), layer.weight)
 layer.weight += 1e-7
def forward(self, x):
x = self.batch_norm1(x)
 self.recalibrate_layer(self.dense1)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 self.recalibrate_layer(self.dense2)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 self.recalibrate_layer(self.dense3)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
make_submission(score_path(path), path )<load_from_csv>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 x_test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head()<data_type_conversions>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 200
BATCH_SIZE = 256
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=800
x_train = x_train.to_numpy()
x_test = x_test.to_numpy()
y_train = y_train.to_numpy()<train_model>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/pytorch-cv-0-0145-lb-0-01839/FOLD{fold}_{seed}.pth", map_location=torch.device('cpu')))
 model.to(DEVICE)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(valid_loss)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 del model
torch.cuda.empty_cache()
gc.collect()
return oof, predictions
x_train = x_train.reshape(-1,28,28)
 x_test = x_test.reshape(-1,28,28)
 print("(Image)Train Inputs: " , x_train.shape)
 print("(Image)Test Inputs: " , x_test.shape )<normalization>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def sharpner(img):
img = Image.fromarray(img.astype('uint8'))
img = img.filter(ImageFilter.UnsharpMask(radius=2, percent=150))
return np.array(img)
 for i in range(x_train.shape[0]):
x_train[i] = sharpner(x_train[i])
 for i in range(x_test.shape[0]):
x_test[i] = sharpner(x_test[i])
 print(x_train.shape)
 print(x_test.shape)
 plt.imshow(x_train[3] )<categorify>
SEED = [0]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
def one_hottie(labels,C):

 One_hot_matrix = tf.one_hot(labels,C)
 return tf.keras.backend.eval(One_hot_matrix)
 y_train = one_hottie(y_train, 10)
 print("Y shape: " + str(y_train.shape))<choose_model_class>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<choose_model_class><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )