Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<import_modules>
warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
path = '.. /input/'
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('loading')
 train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_columns = ['click_id', 'ip', 'app', 'device', 'os', 'channel', 'click_time']
train_df = pd.read_csv(path+"train.csv", dtype=dtypes, skiprows = range(1, 131886954), usecols=train_columns)
 test_df = pd.read_csv(path+"test.csv", dtype=dtypes, usecols=test_columns)
 print('finished loading' )
import argparse
import os
import random
import sys
from typing import Tuple
import matplotlib.pyplot as plt
import numpy as np
import pytorch_lightning as pl
import torch
import l5kit
from l5kit.configs import load_config_data
from l5kit.data import ChunkedDataset, LocalDataManager
from l5kit.dataset import AgentDataset
from l5kit.evaluation import compute_metrics_csv, write_pred_csv
from l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace
from l5kit.geometry import transform_points
from l5kit.rasterization import build_rasterizer
from l5kit.visualization import TARGET_POINTS_COLOR, draw_trajectory
from torch.utils.data import DataLoader
import lyft_loss
import lyft_models
import lyft_utils<define_variables>
def prep_data(d):
print('hour, day, wday.... ')
 d['hour'] = pd.to_datetime(d.click_time ).dt.hour.astype('uint8')
 d['day'] = pd.to_datetime(d.click_time ).dt.day.astype('uint8')
 d['wday'] = pd.to_datetime(d.click_time ).dt.dayofweek.astype('uint8')
 print('grouping by ip-day-hour combination.... ')
 gp = d[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})
 d = d.merge(gp, on=['ip', 'day', 'hour'], how='left')
 del gp; gc.collect()
print('group by ip-app combination.... ')
 gp = d[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 d = d.merge(gp, on=['ip', 'app'], how='left')
 del gp; gc.collect()
print('group by ip-app-os combination.... ')
 gp = d[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 d = d.merge(gp, on=['ip', 'app', 'os'], how='left')
 del gp; gc.collect()
print("vars and d type.... ")
 d['qty'] = d['qty'].astype('uint16')
 d['ip_app_count'] = d['ip_app_count'].astype('uint16')
 d['ip_app_os_count'] = d['ip_app_os_count'].astype('uint16')
 print("label encoding.... ")
 d[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)
 print('dropping')
 d.drop(['click_time', 'ip'], 1, inplace=True)
 return d
ALL_DATA_SIZE = 198474478
VAL_INTERVAL_SAMPLES = 250000
CFG_PATH = ".. /input/lyft-mpred-seresnext26-pretrained/agent_motion_config.yaml"
CSV_PATH = "./submission.csv"
MIN_FRAME_HISTORY = 0
MIN_FRAME_FUTURE = 10
VAL_SELECTED_FRAME =(99,)
 SEED = 42
os.environ["PYTHONHASHSEED"] = str(SEED)
 random.seed(SEED)
 np.random.seed(SEED )<set_options>
train_df = prep_data(train_df)
 test_df = prep_data(test_df)
 print("finished" )
class LyftMpredDatamodule(pl.LightningDataModule):
def __init__(
 self,
l5kit_data_folder: str,
cfg: dict,
batch_size: int = 440,
num_workers: int = 16,
downsample_train: bool = False,
is_test: bool = False,
is_debug: bool = False, 
)-> None:
super().__init__()
os.environ["L5KIT_DATA_FOLDER"] = l5kit_data_folder
self.cfg = cfg
self.batch_size = batch_size
self.num_workers = num_workers
self.downsample_train = downsample_train
self.is_test = is_test
self.is_debug = is_debug
def prepare_data(self):
self.dm = LocalDataManager(None)
 self.rasterizer = build_rasterizer(cfg, self.dm)
 def setup(self):
if self.is_test:
print("test mode setup")
 self.test_path, test_zarr, self.test_dataset = self.load_zarr_dataset(
 loader_name="test_data_loader" 
)
 else:
print("train mode setup")
 self.train_path, train_zarr, self.train_dataset = self.load_zarr_dataset(
 loader_name="train_data_loader" 
)
 self.val_path, val_zarr, self.val_dataset = self.load_zarr_dataset(
 loader_name="val_data_loader" 
)
 self.plot_dataset(self.train_dataset)
 if self.downsample_train:
print(
 "downsampling agents, using only {} frames from each scene".format(
 len(lyft_utils.TRAIN_DSAMPLE_FRAMES)
)
)
 train_agents_list = lyft_utils.downsample_agents(
 train_zarr,
self.train_dataset,
selected_frames=lyft_utils.TRAIN_DSAMPLE_FRAMES, 
)
 self.train_dataset = torch.utils.data.Subset(
 self.train_dataset, train_agents_list 
)
 val_agents_list = lyft_utils.downsample_agents(
 val_zarr, self.val_dataset, selected_frames=VAL_SELECTED_FRAME 
)
 self.val_dataset = torch.utils.data.Subset(
 self.val_dataset, val_agents_list 
)
 def train_dataloader(self):
return DataLoader(
 self.train_dataset,
shuffle=True,
batch_size=self.batch_size,
num_workers=self.num_workers, 
)
 def val_dataloader(self):
return DataLoader(
 self.val_dataset,
shuffle=False,
batch_size=self.batch_size,
num_workers=self.num_workers, 
)
 def test_dataloader(self):
return DataLoader(
 self.test_dataset,
shuffle=False,
batch_size=self.batch_size,
num_workers=self.num_workers, 
)
 def load_zarr_dataset(
 self, loader_name: str = "train_data_loder" 
)-> Tuple[str, ChunkedDataset, AgentDataset]:
zarr_path = self.dm.require(self.cfg[loader_name]["key"])
 print("load zarr data:", zarr_path)
 zarr_dataset = ChunkedDataset(zarr_path ).open()
if loader_name == "test_data_loader":
mask_path = os.path.join(os.path.dirname(zarr_path), "mask.npz")
 agents_mask = np.load(mask_path)["arr_0"]
agent_dataset = AgentDataset(
 self.cfg, zarr_dataset, self.rasterizer, agents_mask=agents_mask 
)
 else:
agent_dataset = AgentDataset(
 self.cfg,
zarr_dataset,
self.rasterizer,
min_frame_history=MIN_FRAME_HISTORY,
min_frame_future=MIN_FRAME_FUTURE, 
)
 print(zarr_dataset)
 return zarr_path, zarr_dataset, agent_dataset
def plot_dataset(self, agent_dataset: AgentDataset, plot_num: int = 10)-> None:
print("Ploting dataset")
 ind = np.random.randint(0, len(agent_dataset), size=plot_num)
 for i in range(plot_num):
data = agent_dataset[ind[i]]
im = data["image"].transpose(1, 2, 0)
 im = agent_dataset.rasterizer.to_rgb(im)
 target_positions_pixels = transform_points(
 data["target_positions"], data["raster_from_agent"] 
)
 draw_trajectory(
 im,
target_positions_pixels,
TARGET_POINTS_COLOR,
yaws=data["target_yaws"], 
)
 plt.imshow(im[::-1])
 if self.is_debug:
plt.show()<init_hyperparams>
RANDOM_SEED = 1
random.seed(RANDOM_SEED)
 np.random.seed(RANDOM_SEED)
 X_train, X_test = train_test_split(train_df, test_size=0.1, random_state=RANDOM_SEED)
 y_train = X_train['is_attributed']
X_train = X_train.drop(['is_attributed'], axis=1)
 y_test = X_test['is_attributed']
X_test = X_test.drop(['is_attributed'], axis=1 )
class LitModel(pl.LightningModule):
def __init__(
 self,
cfg: dict,
num_modes: int = 3,
ba_size: int = 128,
lr: float = 3.0e-4,
backbone_name: str = "efficientnet_b1",
epochs: int = 1,
total_steps: int = 100,
data_size: int = ALL_DATA_SIZE,
optim_name: str = "adam", 
)-> None:
super().__init__()
self.save_hyperparameters(
 "lr",
"backbone_name",
"num_modes",
"ba_size",
"epochs",
"optim_name",
"data_size",
"total_steps", 
)
 self.model = lyft_models.LyftMultiModel(
 cfg, num_modes=num_modes, backbone_name=backbone_name 
)
 self.test_keys =("world_from_agent", "centroid", "timestamp", "track_id")
 def forward(self, x):
x = self.model(x)
 return x
def training_step(self, batch, batch_idx):
inputs = batch["image"]
target_availabilities = batch["target_availabilities"].unsqueeze(-1)
 targets = batch["target_positions"]
outputs, confidences = self.model(inputs)
 loss = lyft_loss.pytorch_neg_multi_log_likelihood_batch(
 targets,
outputs,
confidences.squeeze() ,
target_availabilities.squeeze() , 
)
 self.log(
 "train_epoch_loss",
loss,
prog_bar=False,
on_epoch=True,
on_step=False, 
)
 self.log("train_loss", loss)
 return loss
def validation_step(self, batch, batch_idx):
inputs = batch["image"]
target_availabilities = batch["target_availabilities"].unsqueeze(-1)
 targets = batch["target_positions"]
outputs, confidences = self.model(inputs)
 loss = lyft_loss.pytorch_neg_multi_log_likelihood_batch(
 targets,
outputs,
confidences.squeeze() ,
target_availabilities.squeeze() , 
)
 self.log("val_loss", loss)
 return loss
def test_step(self, batch, batch_idx):
inputs = batch["image"]
outputs, confidences = self.model(inputs)
 test_batch = {key_: batch[key_] for key_ in self.test_keys}
return outputs, confidences, test_batch
def test_epoch_end(self, outputs):

 pred_coords_list = []
confidences_list = []
timestamps_list = []
track_id_list = []
for outputs, confidences, batch in outputs:
outputs = outputs.cpu().numpy()
world_from_agents = batch["world_from_agent"].cpu().numpy()
centroids = batch["centroid"].cpu().numpy()
for idx in range(len(outputs)) :
for mode in range(3):
outputs[idx, mode, :, :] =(
 transform_points(
 outputs[idx, mode, :, :], world_from_agents[idx] 
)
 - centroids[idx][:2] 
)
 pred_coords_list.append(outputs)
 confidences_list.append(confidences)
 timestamps_list.append(batch["timestamp"])
 track_id_list.append(batch["track_id"])
 coords = np.concatenate(pred_coords_list)
 confs = torch.cat(confidences_list ).cpu().numpy()
track_ids = torch.cat(track_id_list ).cpu().numpy()
timestamps = torch.cat(timestamps_list ).cpu().numpy()
write_pred_csv(
 CSV_PATH,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs, 
)
 print(f"Saved to {CSV_PATH}")
 def configure_optimizers(self):
if self.hparams.optim_name == "sgd":
optimizer = torch.optim.SGD(
 self.parameters() ,
lr=self.hparams.lr,
momentum=0.9,
weight_decay=4e-5, 
)
 elif self.hparams.optim_name == "adam":
optimizer = torch.optim.Adam(self.parameters() , lr=self.hparams.lr)
 else:
raise NotImplementedError
scheduler = torch.optim.lr_scheduler.OneCycleLR(
 optimizer, max_lr=self.hparams.lr, total_steps=self.hparams.total_steps 
)
 return [optimizer], [scheduler]<load_pretrained>
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, SpatialDropout1D, Conv1D
from keras.callbacks import Callback
from keras.models import Model
from keras.optimizers import Adam
mpred_dm = LyftMpredDatamodule(
 args.l5kit_data_folder,
cfg,
batch_size=args.batch_size,
num_workers=args.num_workers,
downsample_train=args.downsample_train,
is_test=args.is_test,
is_debug=args.is_debug, 
)
 mpred_dm.prepare_data()
mpred_dm.setup()<load_pretrained>
def get_keras_data(dataset):
X = {
'app': np.array(dataset.app),
'ch': np.array(dataset.channel),
'dev': np.array(dataset.device),
'os': np.array(dataset.os),
'h': np.array(dataset.hour),
'd': np.array(dataset.day),
'wd': np.array(dataset.wday),
'qty': np.array(dataset.qty),
'c1': np.array(dataset.ip_app_count),
'c2': np.array(dataset.ip_app_os_count)
 }
return X
train = get_keras_data(X_train)
 test = get_keras_data(X_test )
if args.is_test:
print("\t\t ==== TEST MODE ====")
 print("load from: ", args.ckpt_path)
 model = LitModel.load_from_checkpoint(args.ckpt_path, cfg=cfg)
 trainer = pl.Trainer(gpus=len(args.visible_gpus.split(",")))
 trainer.test(model, datamodule=mpred_dm )<train_model>
max_app = np.max([train_df['app'].max() , test_df['app'].max() ])+1
max_ch = np.max([train_df['channel'].max() , test_df['channel'].max() ])+1
max_dev = np.max([train_df['device'].max() , test_df['device'].max() ])+1
max_os = np.max([train_df['os'].max() , test_df['os'].max() ])+1
max_h = np.max([train_df['hour'].max() , test_df['hour'].max() ])+1
max_d = np.max([train_df['day'].max() , test_df['day'].max() ])+1
max_wd = np.max([train_df['wday'].max() , test_df['wday'].max() ])+1
max_qty = np.max([train_df['qty'].max() , test_df['qty'].max() ])+1
max_c1 = np.max([train_df['ip_app_count'].max() , test_df['ip_app_count'].max() ])+1
max_c2 = np.max([train_df['ip_app_os_count'].max() , test_df['ip_app_os_count'].max() ])+1
if not args.is_test:
print("\t\t ==== TRAIN MODE ====")
 print(
 "training samples: {}, valid samples: {}".format(
 len(mpred_dm.train_dataset), len(mpred_dm.val_dataset)
)
)
 total_steps = args.epochs * len(mpred_dm.train_dataset)// args.batch_size
val_check_interval = VAL_INTERVAL_SAMPLES // args.batch_size
model = LitModel(
 cfg,
lr=args.lr,
backbone_name=args.backbone_name,
num_modes=args.num_modes,
optim_name=args.optim_name,
ba_size=args.batch_size,
epochs=args.epochs,
data_size=len(mpred_dm.train_dataset),
total_steps=total_steps, 
)
 checkpoint_callback = pl.callbacks.ModelCheckpoint(
 monitor="val_loss",
save_last=True,
mode="min",
verbose=True, 
)
 pl.trainer.seed_everything(seed=SEED)
 trainer = pl.Trainer(
 gpus=len(args.visible_gpus.split(",")) ,
max_steps=total_steps,
val_check_interval=val_check_interval,
precision=args.precision,
benchmark=True,
deterministic=False,
checkpoint_callback=checkpoint_callback, 
)
 trainer.fit(model, datamodule=mpred_dm )<define_variables>
emb_n = 50
in_app = Input(shape=[1], name = 'app')
 emb_app = Embedding(max_app, emb_n )(in_app)
 in_ch = Input(shape=[1], name = 'ch')
 emb_ch = Embedding(max_ch, emb_n )(in_ch)
 in_dev = Input(shape=[1], name = 'dev')
 emb_dev = Embedding(max_dev, emb_n )(in_dev)
 in_os = Input(shape=[1], name = 'os')
 emb_os = Embedding(max_os, emb_n )(in_os)
 in_h = Input(shape=[1], name = 'h')
 emb_h = Embedding(max_h, emb_n )(in_h)
 in_d = Input(shape=[1], name = 'd')
 emb_d = Embedding(max_d, emb_n )(in_d)
 in_wd = Input(shape=[1], name = 'wd')
 emb_wd = Embedding(max_wd, emb_n )(in_wd)
 in_qty = Input(shape=[1], name = 'qty')
 emb_qty = Embedding(max_qty, emb_n )(in_qty)
 in_c1 = Input(shape=[1], name = 'c1')
 emb_c1 = Embedding(max_c1, emb_n )(in_c1)
 in_c2 = Input(shape=[1], name = 'c2')
 emb_c2 = Embedding(max_c2, emb_n )(in_c2)
 fe = concatenate([(emb_app),(emb_ch),(emb_dev),(emb_os),(emb_h), 
(emb_d),(emb_wd),(emb_qty),(emb_c1),(emb_c2)] )
!cp /kaggle/input/lyft-submission-20201126/20201126_ensemble_1_submission.csv./submission.csv<import_modules>
s_dout = SpatialDropout1D(0.25 )(fe)
 fl1 = Flatten()(s_dout)
 conv1 = Conv1D(70, kernel_size=2, strides=1, padding='same' )(s_dout)
 fl2 = Flatten()(conv1)
 conv2 = Conv1D(70, kernel_size=4, strides=1, padding='same' )(s_dout)
 fl3 = Flatten()(conv2)
 concat = concatenate([(fl1),(fl2),(fl3)])
 x = Dropout(0.25 )(Dense(500, activation='relu' )(concat))
x = Dropout(0.35 )(Dense(750, activation='relu' )(x))
x = Dropout(0.45 )(Dense(1000, activation='relu' )(x))
outp = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[in_app,in_ch,in_dev,in_os,in_h,in_d,in_wd,in_qty,in_c1,in_c2], outputs=outp )
warnings.filterwarnings("ignore" )<import_modules>
batch_size = 50000
epochs = 1
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(train_df)/ batch_size)* epochs
lr_init, lr_fin = 0.0025, 0.0001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 optimizer_adam = Adam(lr=0.0025, decay=lr_decay)
 model.compile(loss='binary_crossentropy', optimizer=optimizer_adam, metrics=['accuracy'] )
l5kit.__version__<set_options>
class_weight = {0:.01,1:.99}
model.fit(train, y_train, batch_size=batch_size, epochs=epochs, class_weight=class_weight, shuffle=True, validation_data=(test, y_test), verbose=1 )
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 set_seed(42 )<init_hyperparams>
predictions = model.predict(test, batch_size=batch_size, verbose=1 )
cfg = {
'format_version': 4,
'data_path': "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
'model_params': {
'model_architecture': 'resnet34',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1,
'model_name': "model_resnet34_output",
'lr': 1e-3,
'weight_path': "/kaggle/input/lyft-pretrained-model-hv/model_multi_update_lyft_public.pth",
'train': False,
'predict': True
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 16,
'shuffle': True,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 101,
'checkpoint_every_n_steps': 20,
}
}<feature_engineering>
ids = test_df['click_id']
test_df.drop('click_id', axis=1, inplace=True)
 t = get_keras_data(test_df)
 sub_preds = model.predict(t, batch_size=batch_size, verbose=1 )
<create_dataframe><EOS> 
sub = pd.DataFrame()
sub['click_id'] = ids
sub['is_attributed'] = sub_preds
sub.to_csv('dl_sub.csv', index=False )