Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<predict_on_test>
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )
timestamp, track_id, c, y = predict(agent_ids, batch_size=2000)
 timestamp.shape, track_id.shape, c.shape, y.shape<set_options>
n_comp_GENES = 463
n_comp_CELLS = 60
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
QuantileTransformer_n_quantiles = 500
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )
pd.options.display.max_columns=305<prepare_x_and_y>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
%%time
cols = make_colnames()
df = pd.DataFrame(columns=cols)
 df[cols[0]] = timestamp
df[cols[1]] = track_id
df[cols[2:5]] = c
df[cols[5:]] = y
sel = ~(df[cols[5:]]==0 ).all(1)
 df = df[sel]
df.shape<save_to_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
df.to_csv("submission.csv", index=False )<define_variables>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=QuantileTransformer_n_quantiles,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
xpred = y[:, ::2].reshape(y.shape[0], -1, 3)
 xpred[sel][:3, :10].round(3 )<install_modules>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
!pip install pytorch-pfn-extras==0.3.1<set_options>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50 )<import_modules>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
rc('animation', html='jshtml')
 print("l5kit version:", l5kit.__version__ )<import_modules>
data = train_features.append(test_features)
 data
import torch
from pathlib import Path
import pytorch_pfn_extras as ppe
from math import ceil
from pytorch_pfn_extras.training import IgniteExtensionsManager
from pytorch_pfn_extras.training.triggers import MinValueTrigger
from torch import nn, optim
from torch.utils.data import DataLoader
from torch.utils.data.dataset import Subset
import pytorch_pfn_extras.training.extensions as E<choose_model_class>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
backbone = resnet18(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences 
<load_pretrained>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
def save_yaml(filepath, content, width=120):
with open(filepath, 'w')as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath):
with open(filepath, 'r')as f:
content = yaml.safe_load(f)
 return content
class DotDict(dict):

 __getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__ 
<predict_on_test>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
def run_prediction(predictor, data_loader):
predictor.eval()
pred_coords_list = []
confidences_list = []
timestamps_list = []
track_id_list = []
with torch.no_grad() :
dataiter = tqdm(data_loader)
 for data in dataiter:
image = data["image"].to(device)
 pred, confidences = predictor(image)
 pred_coords_list.append(pred.cpu().numpy().copy())
 confidences_list.append(confidences.cpu().numpy().copy())
 timestamps_list.append(data["timestamp"].numpy().copy())
 track_id_list.append(data["track_id"].numpy().copy())
 timestamps = np.concatenate(timestamps_list)
 track_ids = np.concatenate(track_id_list)
 coords = np.concatenate(pred_coords_list)
 confs = np.concatenate(confidences_list)
 return timestamps, track_ids, coords, confs 
<init_hyperparams>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
cfg = {
'format_version': 4,
'model_params': {
'model_architecture': 'resnet50',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 12,
'shuffle': True,
'num_workers': 4
},
'valid_data_loader': {
'key': 'scenes/validate.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 8,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 10000,
'checkpoint_every_n_steps': 5000,
}
} 
<define_variables>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
flags_dict = {
"debug": False,
"l5kit_data_folder": "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
"pred_mode": "multi",
"device": "cuda:0",
"out_dir": "results/multi_train",
"epoch": 2,
"snapshot_freq": 50,
}<load_pretrained>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
flags = DotDict(flags_dict)
 out_dir = Path(flags.out_dir)
 os.makedirs(str(out_dir), exist_ok=True)
 print(f"flags: {flags_dict}")
 save_yaml(out_dir / 'flags.yaml', flags_dict)
 save_yaml(out_dir / 'cfg.yaml', cfg)
 debug = flags.debug<load_pretrained>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
pass
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
l5kit_data_folder = "/kaggle/input/lyft-motion-prediction-autonomous-vehicles"
os.environ["L5KIT_DATA_FOLDER"] = l5kit_data_folder
dm = LocalDataManager(None)
 print("Load dataset...")
 default_test_cfg = {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
}
test_cfg = cfg.get("test_data_loader", default_test_cfg)
 rasterizer = build_rasterizer(cfg, dm)
 test_path = test_cfg["key"]
print(f"Loading from {test_path}")
 test_zarr = ChunkedDataset(dm.require(test_path)).open()
print("test_zarr", type(test_zarr))
test_mask = np.load(f"{l5kit_data_folder}/scenes/mask.npz")["arr_0"]
test_agent_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataset = test_agent_dataset
if debug:
test_dataset = Subset(test_dataset, np.arange(100))
test_loader = DataLoader(
 test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"],
pin_memory=True, 
)
 print(test_agent_dataset)
 print("
print("<load_pretrained>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
device = torch.device(flags.device)
 if flags.pred_mode == "multi":
predictor = LyftMultiModel(cfg)
 else:
raise ValueError(f"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}")
 pt_path = "/kaggle/input/lyft-resnet18-baseline/0918_predictor_full.pt"
print(f"Loading from {pt_path}")
 predictor.load_state_dict(torch.load(pt_path))
predictor.to(device )<predict_on_test>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
timestamps, track_ids, coords, confs = run_prediction(predictor, test_loader )<save_to_csv>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
csv_path = "submission.csv"
write_pred_csv(
 csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
 print(f"Saved to {csv_path}" )<set_options>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
HTML('<center><iframe width="850" height="450" src="https://www.youtube.com/embed/K0H43N-Hx7w" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></center>' )<install_modules>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
!pip install pymap3d==2.1.0
!pip install -U l5kit<set_options>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/20221107fe-fs-dnn-model/FOLD{fold}_.pth"))
model.to(DEVICE)
 torch.manual_seed(seed)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(valid_loss)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 del model
torch.cuda.empty_cache()
gc.collect()
return oof, predictions
warnings.filterwarnings('ignore')
 sns.set()
%config InlineBackend.figure_format = 'retina'
rc('animation', html='jshtml' )<load_pretrained>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
os.environ["L5KIT_DATA_FOLDER"] = ".. /input/lyft-motion-prediction-autonomous-vehicles"
cfg = load_config_data(".. /input/lyft-config-files/visualisation_config.yaml")
 print(cfg )<load_pretrained>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
dm = LocalDataManager()
dataset_path = dm.require(cfg["val_data_loader"]["key"])
 zarr_dataset = ChunkedDataset(dataset_path)
 zarr_dataset.open()
print(zarr_dataset )<create_dataframe>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
agents = zarr_dataset.agents
probabilities = agents["label_probabilities"]
labels_indexes = np.argmax(probabilities, axis=1)
 counts = []
for idx_label, label in enumerate(PERCEPTION_LABELS):
counts.append(np.sum(labels_indexes == idx_label))
table = PrettyTable(field_names=["label", "counts"])
 for count, label in zip(counts, PERCEPTION_LABELS):
table.add_row([label, count])
 print(table )<create_dataframe>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )