Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
<SOS> metric: PostProcessorKernel Kaggle data source: lyft-motion-prediction-autonomous-vehicles<load_from_csv>
try:
except ModuleNotFoundError:
!pip install --use-feature=2020-resolver zarr > /dev/null
try:
except ModuleNotFoundError:
!pip install --use-feature=2020-resolver pytorch-lightning==0.8.4 > /dev/null
train_data_df = pd.read_csv('.. /input/train.csv')
 test_data_df = pd.read_csv('.. /input/test.csv' )<groupby>
import models.pointnet as pointnet
from training.trainer import get_last_checkpoint, BaseLightningModule
from training.configs.base import GenericConfig
from data.dataset import collate_V5 as collate, CustomLyftDataset_V7 as CustomLyftDataset
new_train_data_df = train_data_df.groupby(['matchId','groupId'])['assists', 'boosts', 'damageDealt', 'DBNOs','headshotKills', 'heals', 'killPlace', 'killPoints', 'kills','killStreaks', 'longestKill', 'maxPlace', 'numGroups', 'revives','rideDistance', 'roadKills', 'swimDistance', 'teamKills','vehicleDestroys', 'walkDistance', 'weaponsAcquired', 'winPoints'].agg('mean' ).reset_index()
new_train_data_df['winPlacePerc'] = train_data_df['winPlacePerc']
new_train_data_df['winPlacePerc'] = train_data_df.groupby(['matchId', 'groupId'])['winPlacePerc'].agg('mean' ).reset_index() ['winPlacePerc']<groupby>
from pathlib import Path
import zarr
from tqdm.notebook import tqdm
import numpy as np, pandas as pd
import torch, torch.nn as nn
from torch.utils.data import DataLoader
new_test_data_df = test_data_df.groupby(['matchId','groupId'])['assists', 'boosts', 'damageDealt', 'DBNOs','headshotKills', 'heals', 'killPlace', 'killPoints', 'kills','killStreaks', 'longestKill', 'maxPlace', 'numGroups', 'revives','rideDistance', 'roadKills', 'swimDistance', 'teamKills','vehicleDestroys', 'walkDistance', 'weaponsAcquired', 'winPoints'].agg('mean' ).reset_index()<import_modules>
import bisect, yaml
import itertools as it
import numpy as np<define_variables>
pool = Parallel(3 )
train_data = train_data_df.values
test_data = test_data_df.values
train_features = new_train_data_df.values[:, 2:24][0:np.int32(0.8*len(new_train_data_df)) ]
train_targets = new_train_data_df.values[:, 24][0:np.int32(0.8*len(new_train_data_df)) ]
val_features = new_train_data_df.values[:, 2:24][np.int32(0.8*len(new_train_data_df)) :len(new_train_data_df)]
val_targets = new_train_data_df.values[:, 24][np.int32(0.8*len(new_train_data_df)) :len(new_train_data_df)]
test_features = new_test_data_df.values[:, 2:24]
test_data = test_data[:, 0:3]<import_modules>
DATA_ROOT = Path(r".. /input/lyft-motion-prediction-autonomous-vehicles")
 ZARR_PATH = Path("scenes/test.zarr")
 HPARAM_PATH = r".. /input/best-submission/hparams.yaml"
print("DATA_ROOT: {}
ZARR_PATH: {}".format(DATA_ROOT, ZARR_PATH))
import sklearn<normalization>
%%time
mask = np.load(DATA_ROOT/"scenes/mask.npz")['arr_0']
mask
scaler = MinMaxScaler(feature_range=(-1, 1)).fit(train_features)
 train_features = scaler.transform(train_features)
 val_features = scaler.transform(val_features)
 test_features = scaler.transform(test_features )<train_model>
agent_ids = np.where(mask)[0]
agent_ids
model = CatBoostRegressor(iterations=20000, learning_rate=0.1, eval_metric='MAE')
 model.fit(train_features, train_targets, eval_set=(val_features, val_targets))<predict_on_test>
with open(HPARAM_PATH)as f:
hparams = yaml.load(f, Loader=yaml.SafeLoader)
 hparams
predictions = model.predict(test_features )<prepare_output>
HBACKWARD = hparams["data_options"]["train"]["kwargs"]["hbackward"]
FRAME_STRIDE = hparams["data_options"]["train"]["kwargs"]["frame_stride"]
AGENT_FEATURE_DIM = hparams["data_options"]["train"]["kwargs"]["agent_feature_dim"]
MAX_AGENTS = hparams["data_options"]["train"]["kwargs"]["max_agents"]
HBACKWARD,FRAME_STRIDE, AGENT_FEATURE_DIM, MAX_AGENTS
new_test_data_df['winPlacePercPred'] = predictions<groupby>
hparams["model_options"]["kwargs"]
group_preds = new_test_data_df.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()<prepare_output>
HFORWARD = 0
NFRAMES = 1
group_preds = group_preds['winPlacePercPred']<define_variables>
z = zarr.open(DATA_ROOT.joinpath(ZARR_PATH ).as_posix() )
dictionary = dict(zip(new_test_data_df['groupId'].values, group_preds))<define_variables>
agents_ij = z.frames["agent_index_interval"]
new_preds = []
for i in test_data_df['groupId'].values:
new_preds.append(dictionary[i] )<prepare_output>
test_data_df['winPlacePercPred'] = new_preds<sort_values>
dt = CustomLyftDataset(
 z,
nframes=NFRAMES,
frame_stride=FRAME_STRIDE,
hbackward=HBACKWARD,
hforward=HFORWARD,
max_agents=MAX_AGENTS,
agent_feature_dim=AGENT_FEATURE_DIM, 
)
 dt.nread_frames
test_data_df.sort_values(by=['matchId', 'groupId'] )<import_modules>
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
import numpy as np<create_dataframe>
model_name = "ListStackedPointNetWithProjection_V3"
model_options = GenericConfig(
 "model",
instance_name=model_name,
kwargs=hparams["model_options"]["kwargs"],
context={"model": getattr(pointnet, model_name)}, 
)
 model_options
predictions = pd.DataFrame(np.transpose(np.array([test_data[:, 0], test_data_df['winPlacePercPred']])))
 predictions.columns = ['Id', 'winPlacePerc']<feature_engineering>
def load_model(path):
model = BaseLightningModule(model_options ).to(DEVICE)
 print(model.load_state_dict(torch.load(path, map_location=DEVICE)["state_dict"]))
model.eval()
return model
predictions['Id'] = np.int32(predictions['Id'] )<save_to_csv>
paths = [
".. /input/best-submission/model_epoch15.ckpt",
".. /input/best-submission/model_epoch45.ckpt",
".. /input/best-submission/model_epoch46.ckpt",
".. /input/best-submission/model_epoch47.ckpt",
".. /input/best-submission/model_epoch50.ckpt",
".. /input/best-submission/model_epoch51.ckpt",
".. /input/best-submission/model_epoch52.ckpt",
".. /input/best-submission/model_epoch53.ckpt",
".. /input/best-submission/model_epoch54.ckpt",
".. /input/best-submission/model_epoch55(1 ).ckpt",
".. /input/best-submission/model_epoch56.ckpt",
".. /input/best-submission/model_epoch57.ckpt",
".. /input/best-submission/model_epoch58.ckpt",
".. /input/best-submission/model_epoch63.ckpt",
".. /input/best-submission/model_epoch73.ckpt",
".. /input/best-submission/model_epoch89.ckpt",
".. /input/best-submission/model_epoch95.ckpt",
".. /input/best-submission/model_epoch99.ckpt"
]
weights = np.array([1,1,1,2,2,3,3,3,4,4,4,4,9,9,9,27,27,27], dtype=np.float32)
 weights /= weights.sum()
print(weights)
 nets = [ load_model(path)for path in paths]
assert len(paths)== len(weights)== len(nets)
 nets[0]
predictions.to_csv('PUBG_preds3.csv', index=False )<load_from_csv>
def make_colnames() :
cols= list(
 it.chain(*[
[f"coord_x0{i}", f"coord_y0{i}", f"coord_x1{i}", f"coord_y1{i}", f"coord_x2{i}", f"coord_y2{i}"]
for i in range(50)]))
return ["timestamp", "track_id"] + ["conf_0", "conf_1", "conf_2"] + cols
df_trn = pd.read_csv('.. /input/pubg-finish-placement-prediction/train.csv', nrows=None)
 df_trn = reduce_mem_usage(df_trn)
 df_tst = pd.read_csv('.. /input/pubg-finish-placement-prediction/test.csv', nrows=None)
 df_tst = reduce_mem_usage(df_tst )<load_from_csv>
def get_agent_data(agent_id):
scene,(frame,frame_id), agent = get_scene(agent_id)
 X, X_light, _, _, le = dt.read_frames(
 scene=scene,
start=frame_id-HBACKWARD,
white_tracks=[agent["track_id"]],
encoder=True 
)
 X_track_id = np.array([agent["track_id"]], dtype=np.int64)
 X_track = np.array([le.labels[agent["track_id"]]], dtype=np.int64)
 X_timestamp = np.array([frame["timestamp"]], dtype=np.int64)
 return X, X_light, X_track, X_track_id, X_timestamp
def collate(inp):
inp = zip(*inp)
 inp = map(np.concatenate, inp)
 inp = map(torch.from_numpy, inp)
 inp = tuple(inp)
 return inp
def get_agents_data(agent_ids):
mapper = delayed(get_agent_data)
 jobs = [mapper(agent_id)for agent_id in agent_ids]
return collate(pool(tqdm(jobs, leave=False)) )
lgbm_trn = pd.read_csv('.. /input/pubg-survivor-kit/oof_lgbm1_reg.csv')
 lgbm_trn.columns = [c if 'winPlacePerc' not in c else c+'Pred' for c in lgbm_trn.columns]
lgbm_tst = pd.read_csv('.. /input/pubg-survivor-kit/sub_lgbm1_reg.csv')
 lgbm_tst.columns = [c if 'winPlacePerc' not in c else c+'Pred' for c in lgbm_tst.columns]<concatenate>
@torch.no_grad()
def predict_one_shot(x, x_light, x_track):
assert len(x)== len(x_light)== len(x_track)
 x, x_light, x_track = x.to(DEVICE), x_light.to(DEVICE), x_track.to(DEVICE)
 try:
c,y = [],[]
for net in nets:
_c, _y = net(( x, x_light))
_c = torch.softmax(_c[torch.arange(len(_c)) , x_track], dim=1 ).cpu().numpy()
_y = _y[torch.arange(len(_c)) , x_track].cpu().numpy()
c.append(_c)
 y.append(_y)
 c = np.stack(c)
 y = np.stack(y)
 c = np.sum(c*weights[:, None, None], axis=0)
 y = np.sum(y*weights[:, None, None], axis=0)
 except Exception as e:
c = np.ones(( len(x), 3)) /3.
 y = np.zeros(( len(x), 300))
raise ValueError() from e
return c,y
@torch.no_grad()
def predict(agent_ids, batch_size=200):
c, y = [], []
x_track_id, x_timestamp = [], []
for i in tqdm(list(range(0, len(agent_ids), batch_size))):
x, x_light, x_track, _x_track_id, _x_timestamp = get_agents_data(agent_ids[i:i+batch_size])
 _c, _y = predict_one_shot(x, x_light, x_track)
 c.append(_c)
 y.append(_y)
 x_track_id.append(_x_track_id)
 x_timestamp.append(_x_timestamp)
 c = np.vstack(c)
 y = np.vstack(y)
 x_track_id = np.concatenate(x_track_id)
 x_timestamp = np.concatenate(x_timestamp)
 return x_timestamp, x_track_id, c, y
df_trn2 = pd.concat([df_trn, lgbm_trn['winPlacePercPred']], axis=1)
 df_tst2 = pd.concat([df_tst, lgbm_tst['winPlacePercPred']], axis=1 )<merge>
timestamp, track_id, c, y = predict(agent_ids, batch_size=2000)
 timestamp.shape, track_id.shape, c.shape, y.shape
df_trn3 = df_trn2.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
df_trn3.columns = [c if 'winPlacePerc' not in c else c+'_Rank' for c in df_trn3.columns]
df_trn3 = df_trn3.merge(df_trn2, how='left', on=['matchId','groupId'] )<compute_test_metric>
pd.options.display.max_columns=305
print('MAE by default: {:.4f}'.format(
 mean_absolute_error(df_trn3['winPlacePerc'], df_trn3['winPlacePercPred'])
)
 )<compute_test_metric>
%%time
cols = make_colnames()
df = pd.DataFrame(columns=cols)
 df[cols[0]] = timestamp
df[cols[1]] = track_id
df[cols[2:5]] = c
df[cols[5:]] = y
sel = ~(df[cols[5:]]==0 ).all(1)
 df = df[sel]
df.shape
print('MAE after group ranking: {:.4f}'.format(
 mean_absolute_error(df_trn3['winPlacePerc'], df_trn3['winPlacePercPred_Rank'])
)
 )<merge>
df.to_csv("submission.csv", index=False )
<drop_column><EOS> 
xpred = y[:, ::2].reshape(y.shape[0], -1, 3)
 xpred[sel][:3, :10].round(3 )