Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
<SOS> metric: PostProcessorKernel Kaggle data source: lyft-motion-prediction-autonomous-vehicles<save_to_csv>
!pip install pytorch-pfn-extras==0.3.1
lgbm_tst.to_csv('sub_lgbm_group_ranked_within_game.csv', index=False )<set_options>
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50 )
warnings.filterwarnings(action='ignore')
 plt.style.use('fivethirtyeight')
 gc.enable()<load_from_csv>
rc('animation', html='jshtml')
 print("l5kit version:", l5kit.__version__ )
%time
train_d=dd.read_csv('.. /input/train_V2.csv',blocksize=25e6)
<load_from_csv>
import torch
from pathlib import Path
import pytorch_pfn_extras as ppe
from math import ceil
from pytorch_pfn_extras.training import IgniteExtensionsManager
from pytorch_pfn_extras.training.triggers import MinValueTrigger
from torch import nn, optim
from torch.utils.data import DataLoader
from torch.utils.data.dataset import Subset
import pytorch_pfn_extras.training.extensions as E
train=pd.read_csv('.. /input/train_V2.csv',nrows=100000)
 test=pd.read_csv(".. /input/test_V2.csv",)
 test_id=test['Id']<count_missing_values>
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
backbone = resnet18(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences
train.isnull().sum()<count_missing_values>
def save_yaml(filepath, content, width=120):
with open(filepath, 'w')as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath):
with open(filepath, 'r')as f:
content = yaml.safe_load(f)
 return content
class DotDict(dict):

 __getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
train.isnull().sum()<groupby>
def run_prediction(predictor, data_loader):
predictor.eval()
pred_coords_list = []
confidences_list = []
timestamps_list = []
track_id_list = []
with torch.no_grad() :
dataiter = tqdm(data_loader)
 for data in dataiter:
image = data["image"].to(device)
 pred, confidences = predictor(image)
 pred_coords_list.append(pred.cpu().numpy().copy())
 confidences_list.append(confidences.cpu().numpy().copy())
 timestamps_list.append(data["timestamp"].numpy().copy())
 track_id_list.append(data["track_id"].numpy().copy())
 timestamps = np.concatenate(timestamps_list)
 track_ids = np.concatenate(track_id_list)
 coords = np.concatenate(pred_coords_list)
 confs = np.concatenate(confidences_list)
 return timestamps, track_ids, coords, confs
train_group=train.groupby('groupId',as_index=False)['winPlacePerc'].agg('mean' )<filter>
cfg = {
'format_version': 4,
'model_params': {
'model_architecture': 'resnet50',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 12,
'shuffle': True,
'num_workers': 4
},
'valid_data_loader': {
'key': 'scenes/validate.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 8,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 10000,
'checkpoint_every_n_steps': 5000,
}
}
train[train['groupId']==2]['winPlacePerc']<drop_column>
flags_dict = {
"debug": False,
"l5kit_data_folder": "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
"pred_mode": "multi",
"device": "cuda:0",
"out_dir": "results/multi_train",
"epoch": 2,
"snapshot_freq": 50,
}
train_corr=train.drop(columns=['Id',"matchId","groupId"])
<feature_engineering>
flags = DotDict(flags_dict)
 out_dir = Path(flags.out_dir)
 os.makedirs(str(out_dir), exist_ok=True)
 print(f"flags: {flags_dict}")
 save_yaml(out_dir / 'flags.yaml', flags_dict)
 save_yaml(out_dir / 'cfg.yaml', cfg)
 debug = flags.debug
train_dbno=pd.DataFrame(train['DBNOs'].value_counts() ,columns=['DBNOs'])
 dbno=train_dbno.iloc[:9,:]
dbno.iloc[8]['DBNOs']=train_dbno.iloc[8:,:].sum() ['DBNOs'] 
<define_variables>
l5kit_data_folder = "/kaggle/input/lyft-motion-prediction-autonomous-vehicles"
os.environ["L5KIT_DATA_FOLDER"] = l5kit_data_folder
dm = LocalDataManager(None)
 print("Load dataset...")
 default_test_cfg = {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
}
test_cfg = cfg.get("test_data_loader", default_test_cfg)
 rasterizer = build_rasterizer(cfg, dm)
 test_path = test_cfg["key"]
print(f"Loading from {test_path}")
 test_zarr = ChunkedDataset(dm.require(test_path)).open()
print("test_zarr", type(test_zarr))
test_mask = np.load(f"{l5kit_data_folder}/scenes/mask.npz")["arr_0"]
test_agent_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataset = test_agent_dataset
if debug:
test_dataset = Subset(test_dataset, np.arange(100))
test_loader = DataLoader(
 test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"],
pin_memory=True, 
)
 print(test_agent_dataset)
 print("
print("
set1=set(i for i in train[(train['kills']>40)&(train['heals']==0)].index.tolist() )<define_variables>
device = torch.device(flags.device)
 if flags.pred_mode == "multi":
predictor = LyftMultiModel(cfg)
 else:
raise ValueError(f"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}")
 pt_path = "/kaggle/input/lyft-resnet18-baseline/0918_predictor_full.pt"
print(f"Loading from {pt_path}")
 predictor.load_state_dict(torch.load(pt_path))
predictor.to(device )
set2=set(i for i in train[(train['move']==0)&(train['kills']>20)].index.tolist() )<define_variables>
timestamps, track_ids, coords, confs = run_prediction(predictor, test_loader )
<define_variables><EOS> 
csv_path = "submission.csv"
write_pred_csv(
 csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
 print(f"Saved to {csv_path}" )