Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<data_type_conversions>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(
 images,
score_threshold=0.25, 
):
predictions = []
images = torch.stack(images ).to(device ).float()
img_scale = torch.tensor([[1.] for _ in range(10)] ).to(device)
 img_size = torch.tensor([(IMG_SIZE, IMG_SIZE)for _ in range(10)] ).to(device)
 for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images,img_scales = img_scale,img_size = img_size)
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df<load_pretrained>
def prepare_altair() :

 vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + v5.SCHEMA_VERSION
vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'
vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION
vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'
noext = "?noext"
paths = {
'vega': vega_url + noext,
'vega-lib': vega_lib_url + noext,
'vega-lite': vega_lite_url + noext,
'vega-embed': vega_embed_url + noext
}
workaround = f
return workaround
def add_autoincrement(render_func):
cache = {}
def wrapped(chart, id="vega-chart", autoincrement=True):
if autoincrement:
if id in cache:
counter = 1 + cache[id]
cache[id] = counter
else:
cache[id] = 0
actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])
 else:
if id not in cache:
cache[id] = 0
actual_id = id
return render_func(chart, id=actual_id)
 return wrapped
@add_autoincrement
def render(chart, id="vega-chart"):

 chart_str =
return HTML(
 chart_str.format(
 id=id,
chart=json.dumps(chart)if isinstance(chart, dict)else chart.to_json(indent=None)
)
)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
workaround = prepare_altair()
HTML("".join((
"<script>",
workaround,
"</script>", 
)) )
models =[
load_net5('.. /input/efficientdetearlierversionf0/best-checkpoint-047epoch.bin'),
load_net5('.. /input/efficientdet5f0/best-checkpoint-028epoch.bin'),
load_net7('.. /input/training-efficientdet-f4/effdet5-cutmix-augmix/best-checkpoint-037epoch.bin'),
]
def make_predictions(images, score_threshold=0.1):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 return [predictions]
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def make_tta_models_predictions(images, score_threshold=0.25):
images = torch.stack(images ).float().cuda()
with torch.no_grad() :
predictions = []
for tta_transform in tta_transforms:
for net in models:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def make_models_predictions(images, score_threshold=0.1):
images = torch.stack(images ).cuda().float()
predictions = []
for net in models:
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
%%time
folder_path = '.. /input/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 fix = {o:n for o, n in zip(test_identity.columns, train_identity.columns)}
test_identity.rename(columns=fix, inplace=True)
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 del train_identity, train_transaction, test_identity, test_transaction, fix
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
def resize_predicitions(predictions):

 for i in range(len(predictions)) :
for j in range(len(predictions[i])) :
predictions[i][j]['boxes']=predictions[i][j]['boxes']*2
return predictions
def effdet_organize(predictions, image_index):
boxes = [(prediction[image_index]['boxes'] ).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
if 0:
for i in range(len(scores)) :
scores[i]=normalize(scores[i])
 return boxes, scores, labels
def yolo_organize(boxes,scores):
boxes_output=[]
for i in range(len(boxes[0])) :
boxes_output.append(boxes[0][i].tolist())
 scores_output=[]
for i in range(len(scores[0])) :
scores_output.append(scores[0][i].cpu().float().item())
 if 0:
scores_output = calibrate(scores_output)
 if 0:
scores_output = normalize(scores_output)
 labels_output = [1]*len(scores_output)
 return [boxes_output], [scores_output], [labels_output]
def run_wbf_ensemble(boxes_effdet, boxes_yolo, scores_effdet, scores_yolo, labels_effdet, labels_yolo, iou_thr=0.6, skip_box_thr=0.5, weights=None):
boxes = boxes_effdet + boxes_yolo
scores = scores_effdet + scores_yolo
labels = labels_effdet + labels_yolo
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
def run_wbf(boxes, scores, labels, iou_thr=0.5, skip_box_thr=0.3, weights=None):
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
def run_wbf_initial(boxes,scores, image_size=1024, iou_thr=0.41, skip_box_thr=0.4, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels<predict_on_test>
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
use_Vcols = ['V'+str(x)for x in v]
Vcols = ['V'+str(x)for x in range(1,340)]
drop_Vcols = list(set(Vcols)- set(use_Vcols))
train.drop(drop_Vcols, axis=1, inplace=True)
 test.drop(drop_Vcols, axis=1, inplace=True )
for images, image_ids, image_size in data_loader:
predictions = make_tta_models_predictions(images)
 predictions=resize_predicitions(predictions)
 for i, image in enumerate(images):
image_id=image_ids[i]
boxes_effdet, scores_effdet, labels_effdet = effdet_organize(predictions, image_index=i)
 boxes, scores, labels = run_wbf(boxes_effdet, scores_effdet, labels_effdet, iou_thr=0.6, skip_box_thr=0.5, weights=None)
 boxes =(boxes/2 ).round().astype(np.int32 ).clip(min=0, max=512)
 fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sample = images[i].permute(1,2,0 ).cpu().numpy()
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 1
color =(0, 0, 1)
 thickness = 2
for box,score in zip(boxes,scores):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(0, 0, 1), 1)
 cv2.putText(sample, '{:.2}'.format(score),(box[0]+np.random.randint(20),box[1]), font, fontScale, color, thickness, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample);
break<load_pretrained>
print(f'There are {train.isnull().any().sum() } columns in train dataset with missing values.' )
sys.path.insert(0, ".. /input/yolov5tta/")
 sys.path.insert(0, ".. /input/configyolov5")
 def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = True
model = torch.load(weights, map_location=device)['model'].to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
print(im0s.shape)
 img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
if True:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres,merge=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
if __name__ == '__main__':
class opt:
weights = ".. /input/train-continuity-yolov5x/weights/best_yolov5_fold0.pt"
img_size = 1024
conf_thres = 0.3
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()
all_path,all_score,all_bboxex = res<predict_on_test>
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
one_value_cols_test = [col for col in test.columns if test[col].nunique() <= 1]
one_value_cols == one_value_cols_test
for images, image_ids, image_sizes in data_loader:
predictions = make_tta_models_predictions(images)
 predictions=resize_predicitions(predictions)
 for i, image in enumerate(images):
image_id=image_ids[i]
for row in range(len(all_path)) :
image_id_yolo = all_path[row].split("/")[-1].split(".")[0]
if image_id_yolo==image_id:
boxes_yolo = all_bboxex[row]
scores_yolo = all_score[row]
boxes_yolo, scores_yolo, labels_yolo = yolo_organize(boxes_yolo,scores_yolo)
 boxes, scores, labels =run_wbf(boxes_yolo, scores_yolo, labels_yolo, iou_thr=0.41, skip_box_thr=0.4, weights=None)
 boxes =(boxes/2 ).round().astype(np.int32 ).clip(min=0, max=512)
 fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sample = images[i].permute(1,2,0 ).cpu().numpy()
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 1
color =(0, 0, 1)
 thickness = 2
for box,score in zip(boxes,scores):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(0, 0, 1), 1)
 cv2.putText(sample, '{:.2}'.format(score),(box[0]+np.random.randint(20),box[1]), font, fontScale, color, thickness, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample);
break<predict_on_test>
print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.')
 print(f'There are {len(one_value_cols_test)} columns in test dataset with one unique value.' )
results = []
for images, image_ids, image_sizes in data_loader:
predictions = make_tta_models_predictions(images)
 predictions=resize_predicitions(predictions)
 for i, image in enumerate(images):
image_id=image_ids[i]
image_size= image_sizes[i]
for row in range(len(all_path)) :
image_id_yolo = all_path[row].split("/")[-1].split(".")[0]
if image_id_yolo==image_id:
boxes_yolo_init = all_bboxex[row]
scores_yolo_init = all_score[row]
boxes_yolo, scores_yolo, labels_yolo = yolo_organize(boxes_yolo_init,scores_yolo_init)
 boxes_effdet, scores_effdet, labels_effdet = effdet_organize(predictions, image_index=i)
 boxes, scores, labels = run_wbf_ensemble(boxes_effdet, boxes_yolo, scores_effdet, scores_yolo, labels_effdet,
labels_yolo, iou_thr=0.41, skip_box_thr=0.4, weights=None)
 boxes =(boxes*(image_size/1024)).round().astype(np.int32 ).clip(min=0, max=image_size-1)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
<save_to_csv>
s1 = train['id_03'].value_counts(dropna=False, normalize=True ).head()
s2 = train['id_04'].value_counts(dropna=False, normalize=True ).head()
s3 = train['id_05'].value_counts(dropna=False, normalize=True ).head()
s4 = train['id_06'].value_counts(dropna=False, normalize=True ).head()
s5 = train['id_09'].value_counts(dropna=False, normalize=True ).head()
s6 = train['id_10'].value_counts(dropna=False, normalize=True ).head()
print(pd.concat([s1, s2, s3, s4, s5, s6], axis = 1))
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head(10 )<predict_on_test>
train['id_11'].value_counts(dropna=False, normalize=True ).head()
for images, image_ids, image_sizes in data_loader:
predictions = make_tta_models_predictions(images)
 predictions=resize_predicitions(predictions)
 for i, image in enumerate(images):
image_id=image_ids[i]
image_size= image_sizes[i]
for row in range(len(all_path)) :
image_id_yolo = all_path[row].split("/")[-1].split(".")[0]
if image_id_yolo==image_id:
boxes_yolo = all_bboxex[row]
scores_yolo = all_score[row]
boxes_yolo, scores_yolo, labels_yolo = yolo_organize(boxes_yolo,scores_yolo)
 boxes_effdet, scores_effdet, labels_effdet = effdet_organize(predictions, image_index=i)
 boxes, scores, labels = run_wbf_ensemble(boxes_effdet, boxes_yolo, scores_effdet, scores_yolo, labels_effdet,
labels_yolo, iou_thr=0.41, skip_box_thr=0.4, weights=None)
 boxes =(boxes/2 ).round().astype(np.int32 ).clip(min=0, max=512)
 fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sample = images[i].permute(1,2,0 ).cpu().numpy()
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 1
color =(0, 0, 1)
 thickness = 2
for box,score in zip(boxes,scores):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(0, 0, 1), 1)
 cv2.putText(sample, '{:.2}'.format(score),(box[0]+np.random.randint(20),box[1]), font, fontScale, color, thickness, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample);
break<load_from_csv>
for i in range(1,16):
if i in [1,2,3,5,9]: continue
train['D'+str(i)] = train['D'+str(i)] - train.TransactionDT/np.float32(24*60*60)
 test['D'+str(i)] = test['D'+str(i)] - test.TransactionDT/np.float32(24*60*60 )
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv" )<drop_column>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
train = train.drop(["Province/State"], axis = "columns" )<data_type_conversions>
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train=train,test=test,verbose=True):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations=['mean'], train_df=train, test_df=test,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,df1=train,df2=test):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(nm,verbose=False)
 print(nm,', ',end='')
 def encode_AG2(main_columns, uids, train_df=train, test_df=test):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )
train["Lat"] = train["Lat"].fillna(12.521)
 train["Long"] = train["Long"].fillna(69.968)
 train["Date"] = train["Date"].apply(lambda x: x.replace("-",""))
train["Date"] = train["Date"].astype(int )<data_type_conversions>
train['Trans_min_mean'] = train['TransactionAmt'] - train['TransactionAmt'].mean()
train['Trans_min_std'] = train['Trans_min_mean'] / train['TransactionAmt'].std()
test['Trans_min_mean'] = test['TransactionAmt'] - test['TransactionAmt'].mean()
test['Trans_min_std'] = test['Trans_min_mean'] / test['TransactionAmt'].std()
test = test.drop(["Province/State"], axis = "columns")
 test["Date"] = test["Date"].apply(lambda x: x.replace("-", ""))
test["Date"] = test["Date"].astype(int)
 test["Lat"] = test["Lat"].fillna(12.521)
 test["Long"] = test["Long"].fillna(69.968 )<prepare_x_and_y>
train['TransactionAmt_log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_log'] = np.log(test['TransactionAmt'] )
x = train[["Lat", "Long", "Date"]]
y = train[["ConfirmedCases", "Fatalities"]]<train_model>
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('std' )
model1 = ExtraTreesClassifier()
model1.fit(x ,y["ConfirmedCases"])
<train_model>
train[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = train['P_emaildomain'].str.split('.', expand=True)
 train[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = train['R_emaildomain'].str.split('.', expand=True)
 test[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = test['P_emaildomain'].str.split('.', expand=True)
 test[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = test['R_emaildomain'].str.split('.', expand=True )
model4 = ExtraTreesClassifier()
model4.fit(x ,y["Fatalities"])
<predict_on_test>
%time
train['cents'] =(train['TransactionAmt'] - np.floor(train['TransactionAmt'])).astype('float32')
 test['cents'] =(test['TransactionAmt'] - np.floor(test['TransactionAmt'])).astype('float32')
 print('cents, ', end='')
 encode_FE(train,test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1','addr1')
 encode_CB('card1_addr1','P_emaildomain')
 encode_FE(train,test,['card1_addr1','card1_addr1_P_emaildomain'])
 encode_AG(['TransactionAmt','D9','D11'],['card1','card1_addr1','card1_addr1_P_emaildomain'],['mean','std'],usena=True )
test_x = test[["Lat", "Long", "Date"]]
cases_predict = model1.predict(test_x)
 fat_predict = model4.predict(test_x)
 ConfirmedCases = pd.DataFrame(cases_predict)
 Fatalities = pd.DataFrame(fat_predict)
 test = pd.concat([test, ConfirmedCases, Fatalities], axis = "columns" )<drop_column>
many_null_cols = [col for col in train.columns if train[col].isnull().sum() / train.shape[0] > 0.9]
many_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.9]
test = test.drop(["Country/Region", "Lat", "Long", "Date"], axis = "columns")
 test.columns = ["ForecastId", "ConfirmedCases", "Fatalities"]<save_to_csv>
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
big_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
test.to_csv("submission.csv", index = False )<load_from_csv>
cols_to_drop = list(set(many_null_cols + many_null_cols_test + big_top_value_cols + big_top_value_cols_test + one_value_cols+ one_value_cols_test))
cols_to_drop.remove('isFraud')
 len(cols_to_drop )
data_train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv" )<data_type_conversions>
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
data_train["Date"] = data_train["Date"].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))
data_train["Date"] = data_train["Date"].apply(lambda x: x.timestamp())
 data_train["Date"] = data_train["Date"].astype(int )<data_type_conversions>
def clean_inf_nan(df):
return df.replace([np.inf, -np.inf], np.nan)
 train = clean_inf_nan(train)
 test = clean_inf_nan(test )
data_train = data_train.drop(['Province/State'],axis=1)
 data_train = data_train.dropna()
data_train.isnull().sum()<load_from_csv>
identity_cat_cols = ['id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18',
'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25',
'id_26', 'id_27', 'id_28', 'id_29', 'id_30', 'id_31', 'id_32',
'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
transaction_cat_cols = ['ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2',
'P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3',
'R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3','P_emaildomain', 'R_emaildomain',
'P_emaildomain_bin', 'R_emaildomain_bin', 'P_emaildomain_suffix', 'R_emaildomain_suffix',
'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9']
cat_cols = identity_cat_cols + transaction_cat_cols
cat_cols = list(set(cat_cols)& set(train.columns))
encoded_cols = ['addr1','card1','card2','card3']
cat_cols = list(set(cat_cols)- set(encoded_cols))
common_cat_cols = [col for col in test.columns if col in cat_cols]
cat_cols == common_cat_cols
data_test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv")
 data_test["Date"] = data_test["Date"].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))
data_test["Date"] = data_test["Date"].apply(lambda x: x.timestamp())
 data_test["Date"] = data_test["Date"].astype(int)
 data_test.isnull().sum()<drop_column>
cat_train = train[common_cat_cols]
num_train = train.drop(cat_cols, axis=1)
 cat_test = test[common_cat_cols]
num_test = test.drop(common_cat_cols, axis=1)
 cat_train.columns == cat_test.columns
data_test.drop('Province/State', axis = 1, inplace = True)
 data_test.head()<prepare_x_and_y>
cat_nunique = list(map(lambda col: cat_train[col].nunique() , common_cat_cols))
d = dict(zip(common_cat_cols, cat_nunique))
sorted(d.items() , key=lambda x:x[1] )
X = data_train[['Lat', 'Long', 'Date']]
Y1 = data_train[['ConfirmedCases']]
Y2 = data_train[['Fatalities']]
X_test = data_test[['Lat', 'Long', 'Date']]<train_model>
cat_train = cat_train.astype('category')
 cat_test = cat_test.astype('category')
 print(cat_train.dtypes )
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X,Y1)
 pred1 = model.predict(X_test)
 pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases_prediction"]<train_model>
low_cardinality_cols = [col for col in common_cat_cols if cat_train[col].nunique() < 10]
high_cardinality_cols = list(set(common_cat_cols)- set(low_cardinality_cols))
print('Categorical columns that will be label encoded: ', low_cardinality_cols)
 print('
Categorical columns that will be target encoded(or dropped eventally): ', high_cardinality_cols )
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X,Y2)
 pred2 = model.predict(X_test)
 pred2 = pd.DataFrame(pred2)
 pred2.columns = ["Death_prediction"]<load_from_csv>
for col in low_cardinality_cols:
LEncoder = LabelEncoder()
LEncoder.fit(list(LE_train[col].astype(str ).values)+ list(LE_test[col].astype(str ).values))
LE_train[col] = LEncoder.transform(list(LE_train[col].astype(str ).values))
LE_test[col] = LEncoder.transform(list(LE_test[col].astype(str ).values))
train = num_train.merge(LE_train, how='left', left_index=True, right_index=True)
 test = num_test.merge(LE_test, how='left', left_index=True, right_index=True)
 print(train[low_cardinality_cols].describe())
 print("---------------")
 print(test[low_cardinality_cols].describe() )
data_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 data_submission.columns
sub_new = data_submission[["ForecastId"]]<concatenate>
concat = pd.concat([pred1,pred2,sub_new],axis=1)
 concat.head()
concat.columns = ['ConfirmedCases', 'Fatalities', 'ForecastId']
concat = concat[['ForecastId','ConfirmedCases', 'Fatalities']]<data_type_conversions>
del num_train, num_test, LE_train, LE_test
concat["ConfirmedCases"] = concat["ConfirmedCases"].astype(int)
 concat["Fatalities"] = concat["Fatalities"].astype(int )<save_to_csv>
cols = list(train.columns)
 for c in list(set(['D6','D7','D8','D9','D12','D13','D14'])& set(cols)) :
cols.remove(c)
 for c in list(set(['C3','M5','id_08','id_33'])& set(cols)) :
cols.remove(c)
 for c in list(set(['card4','id_07','id_14','id_21','id_30','id_32','id_34'])& set(cols)) :
cols.remove(c)
 for c in list(set(['id_'+str(x)for x in range(22,28)])& set(cols)) :
cols.remove(c )
concat.to_csv("submission.csv",index=False )<set_options>
final_train = train[cols].copy()
final_test = test[list(set(cols)- set(['isFraud'])) ].copy()
print(f'Our Final Train dataset has {final_train.shape[0]} rows and {final_train.shape[1]} columns.')
 print(f'Our Final Test dataset has {final_test.shape[0]} rows and {final_test.shape[1]} columns.' )
warnings.filterwarnings('ignore')
 %matplotlib inline
pio.templates.default = 'ggplot2'<load_from_csv>
del train, test
dpath = Path('/kaggle/input/titanic')
 train_raw = pd.read_csv(dpath / 'train.csv')
 test_raw = pd.read_csv(dpath / 'test.csv' )<concatenate>
idxT = X_train.index[:3*len(X_train)//4]
idxV = X_train.index[3*len(X_train)//4:]
full = pd.concat([train_raw.copy(deep=True), test_raw.copy(deep=True)], ignore_index=True )<prepare_x_and_y>
y_train.value_counts() /y_train.shape[0]
def split_data(df):

 df = df.reset_index(drop=True)
 passengerId = df['PassengerId']
df = df.drop(columns=['PassengerId'])
 if 'Survived' in df.columns:
df['Survived'] = df['Survived'].astype('category')
 return(df, passengerId )<drop_column>
print("XGBoost version:", xgb.__version__)
 clf = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train.loc[idxT,cols], y_train[idxT],
early_stopping_rounds=100,
eval_set=[(X_train.loc[idxV,cols],y_train[idxV])],
verbose=50)
def remove_wordy_cols(df):

 df = df.drop(columns=['Name', 'Ticket'])
 return df<feature_engineering>
def fill_fare_mean(df):
df['Fare'] = df['Fare'].fillna(df['Fare'].mean())
 return df<categorify>
def apply_all(df, funs, debug=False):

 for fun in funs:
if debug:
print(f'Apply {fun.__name__}')
 df = fun(df)
 return df<categorify>
best_params = {'bagging_fraction': 0.5818772519688797,
'colsample_bytree': 0.3035307099891744,
'feature_fraction': 0.795967379488282,
'gamma': 0.6896677451866189,
'learning_rate': 0.011336192527320772,
'max_depth': 20,
'min_child_samples': 140,
'num_leaves': 230,
'reg_alpha': 0.06035695642758,
'reg_lambda': 0.012734543098346575,
'subsample': 0.7}
prep_nn_1 = lambda x : apply_all(x, [remove_wordy_cols, fill_fare_mean, split_data])
 train = train_raw.copy(deep=True)
 train, train_pId = prep_nn_1(train)
 display(train.head(3))
display(train_pId.head(3 ).to_frame() )<define_variables>
X_train = final_train.copy()
y_train = X_train.pop('isFraud')
 X_test = final_test.copy()
X_train,X_test = X_train.align(other=X_test,join='left', axis=1 )
dls.show_batch() , learn.loss_func<find_best_params>
print("XGBoost version:", xgb.__version__)
 clf = xgb.XGBClassifier(
 n_estimators=2000,
**best_params,
tree_method='gpu_hist',
verbose=50,
early_stopping_rounds=100 
)
 clf.fit(X_train, y_train)
 y_preds = clf.predict_proba(X_test)[:,1]
<train_model><EOS> 
sub['isFraud'] = y_preds
sub.to_csv('XGB_hypopt_model.csv', index=False )