Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
<SOS> metric: PostProcessorKernel Kaggle data source: lyft-motion-prediction-autonomous-vehicles<define_variables>
HTML('<center><iframe width="850" height="450" src="https://www.youtube.com/embed/K0H43N-Hx7w" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></center>' )
set5=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )<define_variables>
!pip install pymap3d==2.1.0
!pip install -U l5kit
set6=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )<define_variables>
warnings.filterwarnings('ignore')
 sns.set()
%config InlineBackend.figure_format = 'retina'
rc('animation', html='jshtml' )
sets=set1 | set2 | set3 | set4 | set5 | set6<drop_column>
os.environ["L5KIT_DATA_FOLDER"] = ".. /input/lyft-motion-prediction-autonomous-vehicles"
cfg = load_config_data(".. /input/lyft-config-files/visualisation_config.yaml")
 print(cfg )
train.drop(train.index[list(sets)],inplace=True )<drop_column>
dm = LocalDataManager()
dataset_path = dm.require(cfg["val_data_loader"]["key"])
 zarr_dataset = ChunkedDataset(dataset_path)
 zarr_dataset.open()
print(zarr_dataset )
train.drop(['roadkillsrate'],axis=1,inplace=True)
 train.columns.drop('winPlacePerc')==test.columns<feature_engineering>
agents = zarr_dataset.agents
probabilities = agents["label_probabilities"]
labels_indexes = np.argmax(probabilities, axis=1)
 counts = []
for idx_label, label in enumerate(PERCEPTION_LABELS):
counts.append(np.sum(labels_indexes == idx_label))
table = PrettyTable(field_names=["label", "counts"])
 for count, label in zip(counts, PERCEPTION_LABELS):
table.add_row([label, count])
 print(table )
def new_features(df):
df['items']=df['boosts']+df['heals']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['headshotKills_over_kills'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['killPlace_over_maxPlace'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_heals'].fillna(0, inplace=True)
 df['walkDistance_over_heals'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['walkDistance_over_kills'].fillna(0, inplace=True)
 df['walkDistance_over_kills'].replace(np.inf, 0, inplace=True)
 df['teamwork'] = df['assists'] + df['revives']
return df<feature_engineering>
rast = build_rasterizer(cfg, dm)
 dataset = EgoDataset(cfg, zarr_dataset, rast )
train=new_features(train)
 test=new_features(test )<prepare_x_and_y>
z = zarr.open("./dataset.zarr", mode="w", shape=(500,), dtype=np.float32, chunks=(100,))
z[0:150] = np.arange(150)
 print(z.info)
 print(z[:10])
 print(z[::20] )
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = train
else:
print("processing test.csv")
 df = test
target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove('matchType')
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y
x_train, y = feature_engineering(True)
<feature_engineering>
SCENE_DTYPE = [ 
("frame_index_interval", np.int64,(2,)) , 
("host", "<U16"), 
("start_time", np.int64), 
("end_time", np.int64),
]
test,z=feature_engineering(False )<normalization>
FRAME_DTYPE = [ 
("timestamp", np.int64), 
("agent_index_interval", np.int64,(2,)) , 
("traffic_light_faces_index_interval", np.int64,(2,)) , 
("ego_translation", np.float64,(3,)) , 
("ego_rotation", np.float64,(3, 3)) ,
]
scaler=MinMaxScaler(feature_range=(-1,1),copy=False ).fit(x_train)
 x_train=scaler.transform(x_train )<normalization>
os.environ["L5KIT_DATA_FOLDER"] = ".. /input/lyft-motion-prediction-autonomous-vehicles"
dm = LocalDataManager()
sample_path = '.. /input/lyft-motion-prediction-autonomous-vehicles/scenes/sample.zarr'
sample_dataset = ChunkedDataset(sample_path)
 sample_dataset.open()
sample_agents = sample_dataset.agents
sample_agents = pd.DataFrame(sample_agents)
 sample_agents.columns = ["data"]; features = ['centroid', 'extent', 'yaw', 'velocity', 'track_id', 'label_probabilities']
for i, feature in enumerate(features):
sample_agents[feature] = sample_agents['data'].apply(lambda x: x[i])
 sample_agents.drop(columns=["data"],inplace=True)
 sample_agents.head()
x_test=scaler.transform(test )<import_modules>
semantic_map_filepath = dm.require(cfg["raster_params"]["semantic_map_key"])
 dataset_meta = _load_metadata(cfg["raster_params"]["dataset_meta_key"], dm)
 world_to_ecef = np.array(dataset_meta["world_to_ecef"], dtype=np.float64)
 map_api = MapAPI(semantic_map_filepath, world_to_ecef)
 MAP_LAYERS = ["junction", "node", "segment", "lane"]
def element_of_type(elem, layer_name):
return elem.element.HasField(layer_name)
 def get_elements_from_layer(map_api, layer_name):
return [elem for elem in map_api.elements if element_of_type(elem, layer_name)]
class MapRenderer:
def __init__(self, map_api):
self._color_map = dict(drivable_area='
road_segment='
road_block='
lane='
self._map_api = map_api
def render_layer(self, layer_name):
fig = plt.figure(figsize=(10, 10))
ax = fig.add_axes([0, 0, 1, 1])
 def render_lanes(self):
all_lanes = get_elements_from_layer(self._map_api, "lane")
 fig = plt.figure(figsize=(10, 10))
ax = fig.add_axes([0, 0, 1, 1])
 for lane in all_lanes:
self.render_lane(ax, lane)
 return fig, ax
def render_lane(self, ax, lane):
coords = self._map_api.get_lane_coords(MapAPI.id_as_str(lane.id))
self.render_boundary(ax, coords["xyz_left"])
 self.render_boundary(ax, coords["xyz_right"])
 def render_boundary(self, ax, boundary):
xs = boundary[:, 0]
ys = boundary[:, 1]
ax.plot(xs, ys, color=self._color_map["lane"], label="lane")
 renderer = MapRenderer(map_api)
 fig, ax = renderer.render_lanes()
from xgboost.sklearn import XGBRegressor
from sklearn.model_selection import cross_val_score,KFold
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Lasso,LinearRegression
from sklearn import preprocessing
from sklearn.model_selection import RandomizedSearchCV,train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone<compute_train_metric>
multisub = pd.read_csv('.. /input/lyft-motion-prediction-autonomous-vehicles/multi_mode_sample_submission.csv' )
n_folds=2
def mbs(model):
kf=KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(x_train)
 score=-cross_val_score(model,x_train,y,scoring='neg_mean_absolute_error',cv=kf)
 return score 
<choose_model_class>
sub0 = pd.read_csv('.. /input/lyftversion2/submission.csv')
 sub1 = pd.read_csv('.. /input/amacorpensembling-raster-sizes/submission.csv')
 sub2 = pd.read_csv('.. /input/lyft-constant-velocity-extrapolation-baseline/submission.csv' )
lasso=Lasso(alpha=.005,random_state=1 )<define_search_space>
multisub[confs] = [.49,.2,.31]
multisub[conf0] = sub0[conf0]
multisub[conf1] = sub1[conf0]
multisub[conf2] = sub2[conf0]
<choose_model_class><EOS> 
multisub.to_csv('submission.csv', index=False, float_format='%.6g' )