Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
<SOS> metric: PostProcessorKernel Kaggle data source: lyft-motion-prediction-autonomous-vehicles<load_from_csv>
pd.set_option('max_columns', 50 )
items=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/items.csv")
 shops=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/shops.csv")
 cats=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv")
 train=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv")
 test=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/test.csv" )<filter>
print("l5kit version:", l5kit.__version__ )
train = train[(train.item_price < 300000)&(train.item_cnt_day < 1000)]<feature_engineering>
import torch
from torch.utils.data import DataLoader
from torch.utils.data.dataset import Subset
from torchvision.models import resnet18
from torch import nn
from typing import Dict
train = train[train.item_price > 0].reset_index(drop = True)
 train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0<feature_engineering>
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
backbone = resnet18(pretrained=False, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<feature_engineering>
def save_yaml(filepath, content, width=120):
with open(filepath, 'w')as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath):
with open(filepath, 'r')as f:
content = yaml.safe_load(f)
 return content
class DotDict(dict):

 __getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск"<categorify>
def run_prediction(predictor, data_loader):
predictor.eval()
pred_coords_list = []
confidences_list = []
timestamps_list = []
track_id_list = []
with torch.no_grad() :
dataiter = tqdm(data_loader)
 for data in dataiter:
image = data["image"].to(device)
 pred, confidences = predictor(image)
 pred_coords_list.append(pred.cpu().numpy().copy())
 confidences_list.append(confidences.cpu().numpy().copy())
 timestamps_list.append(data["timestamp"].numpy().copy())
 track_id_list.append(data["track_id"].numpy().copy())
 timestamps = np.concatenate(timestamps_list)
 track_ids = np.concatenate(track_id_list)
 coords = np.concatenate(pred_coords_list)
 confs = np.concatenate(confidences_list)
 return timestamps, track_ids, coords, confs
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city)
 shops = shops[["shop_id", "shop_category", "shop_city"]]<feature_engineering>
cfg = {
'format_version': 4,
'model_params': {
'model_architecture': 'resnet50',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1
},
'raster_params': {
'raster_size': [448, 448],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 12,
'shuffle': True,
'num_workers': 4
},
'valid_data_loader': {
'key': 'scenes/validate.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 8,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 10000,
'checkpoint_every_n_steps': 5000,
}
}
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"<categorify>
flags_dict = {
"debug": False,
"l5kit_data_folder": "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
"pred_mode": "multi",
"device": "cuda:0",
"out_dir": "results/multi_train",
"epoch": 2,
"snapshot_freq": 50,
}
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]<categorify>
flags = DotDict(flags_dict)
 out_dir = Path(flags.out_dir)
 os.makedirs(str(out_dir), exist_ok=True)
 print(f"flags: {flags_dict}")
 save_yaml(out_dir / 'flags.yaml', flags_dict)
 save_yaml(out_dir / 'cfg.yaml', cfg)
 debug = flags.debug
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
l5kit_data_folder = "/kaggle/input/lyft-motion-prediction-autonomous-vehicles"
os.environ["L5KIT_DATA_FOLDER"] = l5kit_data_folder
dm = LocalDataManager(None)
 print("Load dataset...")
 default_test_cfg = {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
}
test_cfg = cfg.get("test_data_loader", default_test_cfg)
 rasterizer = build_rasterizer(cfg, dm)
 test_path = test_cfg["key"]
print(f"Loading from {test_path}")
 test_zarr = ChunkedDataset(dm.require(test_path)).open()
print("test_zarr", type(test_zarr))
test_mask = np.load(f"{l5kit_data_folder}/scenes/mask.npz")["arr_0"]
test_agent_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataset = test_agent_dataset
if debug:
test_dataset = Subset(test_dataset, np.arange(100))
test_loader = DataLoader(
 test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"],
pin_memory=True, 
)
 print(test_agent_dataset)
 print("
print("
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<feature_engineering>
device = torch.device(flags.device)
 if flags.pred_mode == "multi":
predictor = LyftMultiModel(cfg)
 else:
raise ValueError(f"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}")
 pt_path = "/kaggle/input/lyft-prediction-public-models/multi_mode_448px.pth"
print(f"Loading from {pt_path}")
 state_dict = torch.load(pt_path)
 state_dict_v2 = copy.deepcopy(state_dict)
 for key in state_dict:
new_key = key[10:]
state_dict_v2[new_key] = state_dict_v2.pop(key)
 predictor.load_state_dict(state_dict_v2)
 predictor.to(device )
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"<groupby>
timestamps, track_ids, coords, confs = run_prediction(predictor, test_loader )
<categorify><EOS> 
csv_path = "submission.csv"
write_pred_csv(
 csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
 print(f"Saved to {csv_path}" )