Kaggle competition: Lyft Motion Prediction for Autonomous Vehicles
<SOS> metric: PostProcessorKernel Kaggle data source: lyft-motion-prediction-autonomous-vehicles<concatenate>
warnings.filterwarnings("ignore" )
xy_cols = [[],[],[]]
for i in range(50):
for j in range(3):
xy_cols[j].append(f"coord_x{j}{i}")
 xy_cols[j].append(f"coord_y{j}{i}")
 xy_cols[0][:10]<define_variables>
l5kit.__version__
COLUMNS = ["timestamp", "track_id"] + list(conf_cols)+ xy_cols[0] + xy_cols[1] + xy_cols[2]<prepare_output>
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 set_seed(42 )
def sort_df(df, sort_timestamp_track_id=True):
conf_orders = np.argsort(-df[conf_cols].values,1)
 XY = np.stack([df[xy_cols[0]].values,df[xy_cols[1]].values, df[xy_cols[2]].values], axis=1)
 XY = XY[np.arange(len(XY)) [:, None], conf_orders]
df2 = pd.DataFrame(columns = COLUMNS)
 df2["timestamp"] = df["timestamp"].values
df2["track_id"] = df["track_id"].values
df2[xy_cols[0] + xy_cols[1] + xy_cols[2]] = XY.reshape(-1,300)
 df2[conf_cols] = df[conf_cols].values[np.arange(len(df)) [:, None], conf_orders]
if sort_timestamp_track_id:
df2.sort_values(["timestamp", "track_id"], inplace=True)
 df2.reset_index(inplace=True, drop=True)
 return df2<load_from_csv>
cfg = {
'format_version': 4,
'data_path': "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
'model_params': {
'model_architecture': 'resnet34',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.5,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.5,
'model_name': "model_resnet34_output",
'lr': 1e-3,
'weight_path': "/kaggle/input/lyft-pretrained-model-hv/model_multi_update_lyft_public.pth",
'train': False,
'predict': True
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 16,
'shuffle': True,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 101,
'checkpoint_every_n_steps': 20,
}
}
%%time
df = None
for path,w in zip(paths,weights):
print(w, path)
 temp = pd.read_csv(path)
 temp = sort_df(temp)
 temp[COLUMNS[5:]] *= w
if df is None:
df = temp
else:
df[COLUMNS[2:]] += temp[COLUMNS[2:]]
df[conf_cols] /= df[conf_cols].sum(1 ).values[:, None]
sample = pd.read_csv(".. /input/lyft-motion-prediction-autonomous-vehicles/multi_mode_sample_submission.csv")
 df = sample[["timestamp", "track_id"]].merge(df, on=["timestamp", "track_id"])
 sample.shape, df.shape<save_to_csv>
DIR_INPUT = cfg["data_path"]
os.environ["L5KIT_DATA_FOLDER"] = DIR_INPUT
dm = LocalDataManager(None )
df.to_csv("submission.csv", index=False, float_format='%.6f' )<categorify>
train_cfg = cfg["train_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 train_zarr = ChunkedDataset(dm.require(train_cfg["key"])).open()
train_dataset = AgentDataset(cfg, train_zarr, rasterizer)
 train_dataloader = DataLoader(train_dataset, shuffle=train_cfg["shuffle"], batch_size=train_cfg["batch_size"],
num_workers=train_cfg["num_workers"])
 print("==================================TRAIN DATA==================================")
 print(train_dataset )
random.seed(2016)
 def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 for i, feat in enumerate(features):
outfile.write('{0}\t{1}\tq
'.format(i, feat))
outfile.close()
def get_importance(gbm, features):
create_feature_map(features)
 importance = gbm.get_fscore(fmap='xgb.fmap')
 importance = sorted(importance.items() , key=itemgetter(1), reverse=True)
 return importance
def intersect(a, b):
return list(set(a)& set(b))
def get_features(train, test):
trainval = list(train.columns.values)
 testval = list(test.columns.values)
 output = intersect(trainval, testval)
 output.remove('people_id')
 return sorted(output)
 def run_single(train, test, features, target, random_state=0):
eta = 1.3
max_depth = 3
subsample = 0.8
colsample_bytree = 0.8
start_time = time.time()
print('XGBoost params.ETA: {}, MAX_DEPTH: {}, SUBSAMPLE: {}, COLSAMPLE_BY_TREE: {}'.format(eta, max_depth, subsample, colsample_bytree))
params = {
"objective": "binary:logistic",
"booster" : "gbtree",
"eval_metric": "auc",
"eta": eta,
"tree_method": 'exact',
"max_depth": max_depth,
"subsample": subsample,
"colsample_bytree": colsample_bytree,
"silent": 1,
"seed": random_state,
}
num_boost_round = 115
early_stopping_rounds = 10
test_size = 0.1
X_train, X_valid = train_test_split(train, test_size=test_size, random_state=random_state)
 print('Length train:', len(X_train.index))
print('Length valid:', len(X_valid.index))
y_train = X_train[target]
y_valid = X_valid[target]
dtrain = xgb.DMatrix(X_train[features], y_train)
 dvalid = xgb.DMatrix(X_valid[features], y_valid)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=True)
 print("Validating...")
 check = gbm.predict(xgb.DMatrix(X_valid[features]), ntree_limit=gbm.best_iteration+1)
 score = roc_auc_score(X_valid[target].values, check)
 print('Check error value: {:.6f}'.format(score))
imp = get_importance(gbm, features)
 print('Importance array: ', imp)
 print("Predict test set...")
 test_prediction = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_iteration+1)
 print('Training time: {} minutes'.format(round(( time.time() - start_time)/60, 2)))
 return test_prediction.tolist()
def simple_load() :
print("Read people.csv...")
 people = pd.read_csv(".. /input/people.csv",
dtype={'people_id': np.str,
'activity_id': np.str,
'char_38': np.int32},
parse_dates=['date'])
 print("Load train.csv...")
 train = pd.read_csv(".. /input/act_train.csv",
dtype={'people_id': np.str,
'activity_id': np.str,
'outcome': np.int8},
parse_dates=['date'])
 print("Load test.csv...")
 test = pd.read_csv(".. /input/act_test.csv",
dtype={'people_id': np.str,
'activity_id': np.str},
parse_dates=['date'])
 print("Process tables...")
 for table in [train, test]:
table['activity_category'] = table['activity_category'].str.lstrip('type ' ).astype(np.int32)
 for i in range(1, 11):
table['char_' + str(i)].fillna('type -999', inplace=True)
 table['char_' + str(i)] = table['char_' + str(i)].str.lstrip('type ' ).astype(np.int32)
 people['year'] = people['date'].dt.year
people['month'] = people['date'].dt.month
people['day'] = people['date'].dt.day
people['weekday'] = people['date'].dt.weekday
people['weekend'] =(( people.weekday == 0)|(people.weekday == 6)).astype(int)
 people.drop('date', axis=1, inplace=True)
 people['group_1'] = people['group_1'].str.lstrip('group ' ).astype(np.int32)
 for i in range(1, 10):
people['char_' + str(i)] = people['char_' + str(i)].str.lstrip('type ' ).astype(np.int32)
 for i in range(10, 38):
people['char_' + str(i)] = people['char_' + str(i)].astype(np.int32)
 print("Merge...")
 train = train.merge(people, on="people_id", suffixes=("_act", ""))
test = test.merge(people, on="people_id", suffixes=("_act", ""))
train = train.set_index("activity_id")
 test = test.set_index("activity_id")
 return train, test
def group_decision(train, test, only_certain=True):
lookup = train.groupby(["group_1", "date_act"], as_index=False)["outcome"].mean()
test = pd.merge(test.reset_index() , lookup, how="left", on=["group_1", "date_act"] ).set_index("activity_id")
 train["date_act_fillfw"] = train["date_act"]
train["date_act_fillbw"] = train["date_act"]
train["group_fillfw"] = train["group_1"]
train["group_fillbw"] = train["group_1"]
df = train.append(test)
 df = df.sort_values(by=["group_1", "date_act"])
 df["date_act_fillfw"] = df["date_act_fillfw"].fillna(method="ffill")
 df["date_act_fillbw"] = df["date_act_fillbw"].fillna(method="bfill")
 df["outcome_fillfw"] = df["outcome"].fillna(method="ffill")
 df["outcome_fillbw"] = df["outcome"].fillna(method="bfill")
 df["group_fillfw"] = df["group_fillfw"].fillna(method="ffill")
 df["group_fillbw"] = df["group_fillbw"].fillna(method="bfill")
 df["fw_same_date"] =(df["date_act_fillfw"] == df["date_act"] ).astype(int)
 df["bw_same_date"] =(df["date_act_fillbw"] == df["date_act"] ).astype(int)
 df["fw_same_group"] =(df["group_fillfw"] == df["group_1"] ).astype(int)
 df["bw_same_group"] =(df["group_fillbw"] == df["group_1"] ).astype(int)
 df["interfill"] =(df["outcome_fillfw"] *
df["fw_same_group"] +
df["outcome_fillbw"] *
df["bw_same_group"])/(df["fw_same_group"] +
df["bw_same_group"])
 df["needs cushion"] =(df["fw_same_group"] * df["bw_same_group"] - 1 ).abs()
df["cushion"] = df["needs cushion"] * df["interfill"] * -0.1 + df["needs cushion"] * 0.05
df["interfill"] = df["interfill"] + df["cushion"]
df["outcome"] = df["outcome"].fillna(df["interfill"])
 if only_certain == True:
df = df[(df["outcome"] == 0.0)|(df["outcome"] == 1.0)]
test["outcome"] = df["outcome"]
return test["outcome"]
def xgboost_return(train,test,features):
print("Process tables...")
 for table in [train, test]:
table['year'] = table['date'].dt.year
table['month'] = table['date'].dt.month
table['day'] = table['date'].dt.day
table['weekday'] = table['date'].dt.weekday
table['weekend'] =(( table.weekday == 0)|(table.weekday == 6)).astype(int)
 table.drop('date', axis=1, inplace=True)
 features.remove('date')
 features.remove('date_act')
 test["extra outcomes"] = run_single(train,test,features,"outcome")
 return test["extra outcomes"]
def model() :
train, test = simple_load()
features = get_features(train,test)
 test["outcome"] = group_decision(train, test, only_certain=False)
 test.reset_index() [["activity_id", "outcome"]].to_csv("starter_template.csv", index=False)
 test["outcome"] = test["outcome"].fillna(xgboost_return(train,test,features))
return test.reset_index() [["activity_id", "outcome"]]
def main() :
model().to_csv("submission.csv", index=False)
 if __name__ == "__main__":
main()<load_from_csv>
test_cfg = cfg["test_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open()
test_mask = np.load(f"{DIR_INPUT}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataloader = DataLoader(test_dataset,shuffle=test_cfg["shuffle"],batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
 print("==================================TEST DATA==================================")
 print(test_dataset )
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )<feature_engineering>
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
architecture = cfg["model_params"]["model_architecture"]
backbone = eval(architecture )(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 if architecture == "resnet50":
backbone_out_features = 2048
else:
backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
def forward(data, model, device, criterion = pytorch_neg_multi_log_likelihood_batch):
inputs = data["image"].to(device)
 target_availabilities = data["target_availabilities"].to(device)
 targets = data["target_positions"].to(device)
 preds, confidences = model(inputs)
 loss = criterion(targets, preds, confidences, target_availabilities)
 return loss, preds, confidences
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = LyftMultiModel(cfg)
 weight_path = cfg["model_params"]["weight_path"]
if weight_path:
model.load_state_dict(torch.load(weight_path))
model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg["model_params"]["lr"])
 print(f'device {device}' )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
print(model )
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
if cfg["model_params"]["train"]:
tr_it = iter(train_dataloader)
 progress_bar = tqdm(range(cfg["train_params"]["max_num_steps"]))
num_iter = cfg["train_params"]["max_num_steps"]
losses_train = []
iterations = []
metrics = []
times = []
model_name = cfg["model_params"]["model_name"]
start = time.time()
for i in progress_bar:
try:
data = next(tr_it)
 except StopIteration:
tr_it = iter(train_dataloader)
 data = next(tr_it)
 model.train()
torch.set_grad_enabled(True)
 loss, _, _ = forward(data, model, device)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
losses_train.append(loss.item())
 progress_bar.set_description(f"loss: {loss.item() } loss(avg): {np.mean(losses_train)}")
 if i % cfg['train_params']['checkpoint_every_n_steps'] == 0:
torch.save(model.state_dict() , f'{model_name}_{i}.pth')
 iterations.append(i)
 metrics.append(np.mean(losses_train))
times.append(( time.time() -start)/60)
 results = pd.DataFrame({'iterations': iterations, 'metrics(avg)': metrics, 'elapsed_time(mins)': times})
 results.to_csv(f"train_metrics_{model_name}_{num_iter}.csv", index = False)
 print(f"Total training time is {(time.time() -start)/60} mins")
 print(results.head() )
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
pred_path = 'submission1.csv'
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
 )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
import pandas as pd, numpy as np
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
df = pd.read_csv("./submission1.csv" )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
df_issa1 = df.copy()
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
pd.options.display.max_columns=305
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
df = pd.read_csv(".. /input/lyft-test-set-as-csv/Lyft_test_set.csv")
 print("df.shape:", df.shape)
 df.head(10 )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
def get_models(path):
models = {}
path = Path(path)
 for model in path.glob("lgbm*"):
model_name = get_model_name(model.stem)
 shift = int(model_name.split("shift_")[1])
 meta = path.joinpath("meta_shift_{:02d}.json".format(shift))
with meta.open() as f:
train_cols = json.load(f)["TRAIN_COLS"]
models[model_name] = {"model": model.as_posix() , "train_cols": train_cols}
return models
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
models = get_models(".. /input/lyft-models/lgbm_06")
 len(models )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def make_colnames() :
xcols = ["coord_x{}{}".format(step, rank)for step in range(3)for rank in range(50)]
ycols = ["coord_y{}{}".format(step, rank)for step in range(3)for rank in range(50)]
cols = ["timestamp", "track_id"] + ["conf_0", "conf_1", "conf_2"] + list(it.chain(*zip(xcols, ycols)))
 return cols
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def predict(models, df):
sub = np.empty(( len(df), 305))
sub.fill(np.nan)
 sub = pd.DataFrame(sub, columns = make_colnames())
 sub[["timestamp", "track_id"]] = df[["timestamp", "track_id"]]
sub["conf_0"] = 1.0
for shift in range(1, 51):
for suffix in ["x", "y"]:
model_info = models["lgbm_{}_shift_{:02d}".format(suffix, shift)]
model = lgb.Booster(model_file= model_info["model"])
 pred = model.predict(df[model_info["train_cols"]])
 sub["coord_{}0{}".format(suffix, shift-1)] = pred
if not shift%10:
print("shift: {}".format(shift))
sub.fillna(0., inplace=True)
 return sub
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
sub = predict(models, df )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
sub.to_csv("submission2.csv", index=False )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df_issa2 = sub.copy()
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
import pandas as pd, numpy as np
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df_issa1.to_csv('df_issa1.csv', index=False)
 df_issa2.to_csv('df_issa2.csv', index=False )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
x = 0.7
df = df_issa1.copy()
df['timestamp'] = x*df_issa1['timestamp'] +(1-x)*df_issa2['timestamp']
df['track_id'] = x*df_issa1['track_id'] +(1-x)*df_issa2['track_id']
df.head()
<merge><EOS> 
df.to_csv('submission.csv', index=False )