Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<create_dataframe>
VAL_FILE = '.. /input/training-and-validation-data-pickle/validation.pkl.gz'
test_cfg = cfg["test_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open()
test_mask = np.load(f"{DIR_INPUT}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataloader = DataLoader(test_dataset,shuffle=test_cfg["shuffle"],batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
 print("==================================TEST DATA==================================")
 print(test_dataset )<choose_model_class>
almost_zero = 1e-10
almost_one = 1 - almost_zero
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
architecture = cfg["model_params"]["model_architecture"]
backbone = eval(architecture )(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 if architecture == "resnet50":
backbone_out_features = 2048
else:
backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences<compute_train_metric>
base_models = {
'lgb1 ': "Python LGBM based on Pranav Pandya's R version",
'wbftl': "anttip's Wordbatch FM-FTRL",
'nngpu': "Downampled Neural Network run on GPU"
}
def forward(data, model, device, criterion = pytorch_neg_multi_log_likelihood_batch):
inputs = data["image"].to(device)
 target_availabilities = data["target_availabilities"].to(device)
 targets = data["target_positions"].to(device)
 preds, confidences = model(inputs)
 loss = criterion(targets, preds, confidences, target_availabilities)
 return loss, preds, confidences<choose_model_class>
cvfiles = {
'lgb1 ': '.. /input/validate-pranav-lgb-model/pranav_lgb_val_nostop.csv',
'wbftl': '.. /input/validate-anttip-s-wordbatch-fm-ftrl-9711-version/wordbatch_fm_ftrl_val.csv',
'nngpu': '.. /input/gpu-validation/gpu_val1.csv'
}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = LyftMultiModel(cfg)
 weight_path = cfg["model_params"]["weight_path"]
if weight_path:
model.load_state_dict(torch.load(weight_path))
model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg["model_params"]["lr"])
 print(f'device {device}' )<train_model>
subfiles = {
'lgb1 ': '.. /input/try-pranav-s-r-lgbm-in-python/sub_lgbm_r_to_python_nocv.csv',
'wbftl': '.. /input/anttip-s-wordbatch-fm-ftrl-9711-version/wordbatch_fm_ftrl.csv',
'nngpu': '.. /input/talkingdata-gpu-example-with-multiple-runs/gpu_test2.csv'
}
print(model )<init_hyperparams>
lbscores = {
'lgb1 ':.9694,
'wbftl':.9711,
'nngpu':.9678
}
if cfg["model_params"]["train"]:
tr_it = iter(train_dataloader)
 progress_bar = tqdm(range(cfg["train_params"]["max_num_steps"]))
num_iter = cfg["train_params"]["max_num_steps"]
losses_train = []
iterations = []
metrics = []
times = []
model_name = cfg["model_params"]["model_name"]
start = time.time()
for i in progress_bar:
try:
data = next(tr_it)
 except StopIteration:
tr_it = iter(train_dataloader)
 data = next(tr_it)
 model.train()
torch.set_grad_enabled(True)
 loss, _, _ = forward(data, model, device)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
losses_train.append(loss.item())
 progress_bar.set_description(f"loss: {loss.item() } loss(avg): {np.mean(losses_train)}")
 if i % cfg['train_params']['checkpoint_every_n_steps'] == 0:
torch.save(model.state_dict() , f'{model_name}_{i}.pth')
 iterations.append(i)
 metrics.append(np.mean(losses_train))
times.append(( time.time() -start)/60)
 results = pd.DataFrame({'iterations': iterations, 'metrics(avg)': metrics, 'elapsed_time(mins)': times})
 results.to_csv(f"train_metrics_{model_name}_{num_iter}.csv", index = False)
 print(f"Total training time is {(time.time() -start)/60} mins")
 print(results.head() )<save_to_csv>
cvdata = pd.DataFrame({
m:pd.read_csv(cvfiles[m])['is_attributed'].clip(almost_zero,almost_one ).apply(logit)
 for m in base_models
})
 X_train = np.array(cvdata)
 y_train = pd.read_pickle(VAL_FILE)['is_attributed']
pred_path = 'submission.csv'
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
 )<import_modules>
stack_model = LogisticRegression()
stack_model.fit(X_train, y_train)
 stack_model.coef_
import pandas as pd
import tensorflow as tf
import numpy as np
from kaggle_datasets import KaggleDatasets
from l5kit.evaluation import write_pred_csv
import time<load_from_csv>
weights = stack_model.coef_/stack_model.coef_.sum()
scores = [ roc_auc_score(y_train, expit(cvdata[c])) for c in cvdata.columns ]
names = [ base_models[c] for c in cvdata.columns ]
lb = [ lbscores[c] for c in cvdata.columns ]
pd.DataFrame(data={'LB score': lb, 'CV score':scores, 'weight':weights.reshape(-1)}, index=names )
sub = pd.read_csv('.. /input/lyft-motion-prediction-autonomous-vehicles/multi_mode_sample_submission.csv')
 TEST_DATA_DIR = 'lyft-test-tfrecords'
TEST_GCS_PATH = KaggleDatasets().get_gcs_path(TEST_DATA_DIR)
 test_files = tf.io.gfile.glob(TEST_GCS_PATH + '/test' +'/shard*.tfrecord' )<feature_engineering>
print('Stacker score: ', roc_auc_score(y_train, stack_model.predict_proba(X_train)[:,1]))
feature_descriptions = {
'image': tf.io.FixedLenFeature([], tf.string),
'target_positions': tf.io.FixedLenFeature([], tf.string),
'target_availabilities': tf.io.FixedLenFeature([], tf.string),
'target_yaws': tf.io.FixedLenFeature([], tf.string),
'world_from_agent' : tf.io.FixedLenFeature([], tf.string),
'history_positions': tf.io.FixedLenFeature([], tf.string),
'history_yaws': tf.io.FixedLenFeature([], tf.string),
'history_availabilities': tf.io.FixedLenFeature([], tf.string),
'world_to_image': tf.io.FixedLenFeature([], tf.string),
'track_id': tf.io.FixedLenFeature([], tf.string),
'timestamp': tf.io.FixedLenFeature([], tf.string),
'centroid': tf.io.FixedLenFeature([], tf.string),
'yaw': tf.io.FixedLenFeature([], tf.string),
'extent': tf.io.FixedLenFeature([], tf.string),
}
feature_dtypes = {
'image': tf.uint8,
'target_positions': tf.float32,
'target_availabilities': tf.float32,
'target_yaws': tf.float32,
'world_from_agent' : tf.float64,
'history_positions': tf.float32,
'history_yaws':tf.float32,
'history_availabilities': tf.float32,
'world_to_image': tf.float64,
'track_id': tf.int64,
'timestamp': tf.int64,
'centroid': tf.float64,
'yaw': tf.float64,
'extent': tf.float32,
}<define_variables>
final_sub = pd.DataFrame()
subs = {m:pd.read_csv(subfiles[m] ).rename({'is_attributed':m},axis=1)for m in base_models}
first_model = list(base_models.keys())[0]
final_sub['click_id'] = subs[first_model]['click_id']
AUTO = tf.data.experimental.AUTOTUNE
GLOBAL_BATCH_SIZE = 64*strategy.num_replicas_in_sync
IMG_SIZE = 224
CHANNEL_DIM = 25
TOTAL_STEPS = np.int(np.ceil(len(sub)/ GLOBAL_BATCH_SIZE))<categorify>
df = subs[first_model]
for m in subs:
if m != first_model:
df = df.merge(subs[m], on='click_id')
 df.head()
def padded_batch(images, timestamp , track_id, world_from_agent, target, avail):
if tf.shape(timestamp)[0] != GLOBAL_BATCH_SIZE:
images = tf.concat([images , tf.zeros(( GLOBAL_BATCH_SIZE - tf.shape(images)[0] , IMG_SIZE, IMG_SIZE, CHANNEL_DIM)) ] , axis = 0)
 timestamp = tf.concat([timestamp , tf.zeros(( GLOBAL_BATCH_SIZE - tf.shape(timestamp)[0]), tf.int64)] , axis = 0)
 track_id = tf.concat([track_id , tf.zeros(( GLOBAL_BATCH_SIZE - tf.shape(track_id)[0]), dtype = tf.int64)] , axis = 0)
 world_from_agent = tf.concat([world_from_agent , tf.zeros(( GLOBAL_BATCH_SIZE - tf.shape(world_from_agent)[0] , 3, 3), tf.float64)] , axis = 0)
 return images , timestamp , track_id , world_from_agent , target , avail
def read_unlabeled_tfrecord(example):
example = tf.io.parse_single_example(example, feature_descriptions)
 data = {k:tf.io.parse_tensor(example[k], feature_dtypes[k])for k in example}
image = tf.image.convert_image_dtype(data['image'], dtype = tf.float32)
 image = tf.transpose(tf.squeeze(image), [1,2,0])
 image = tf.reshape(image,(IMG_SIZE, IMG_SIZE, CHANNEL_DIM))
timestamp = tf.squeeze(data['timestamp'])
 track_id = tf.squeeze(data['track_id'])
 world_from_agent = tf.squeeze(data['world_from_agent'])
 target = tf.squeeze(data['target_positions'])
 avail = tf.squeeze(data['target_availabilities'])
 return image , timestamp , track_id , world_from_agent , target, avail
def load_dataset(filenames, ordered = True, training = False):
dataset = tf.data.TFRecordDataset(filenames,compression_type = 'GZIP')
 dataset = dataset.map(read_unlabeled_tfrecord, num_parallel_calls = AUTO ).batch(GLOBAL_BATCH_SIZE,)
 dataset = dataset.map(padded_batch , num_parallel_calls = AUTO)
 return dataset<create_dataframe>
X_test = np.array(df.drop(['click_id'],axis=1 ).clip(almost_zero,almost_one ).apply(logit))
final_sub['is_attributed'] = stack_model.predict_proba(X_test)[:,1]
final_sub.head(10 )
<categorify><EOS> 
final_sub.to_csv("sub_stacked.csv", index=False, float_format='%.9f' )