Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<data_type_conversions>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
X_train['day'] = X_train.TransactionDT /(24*60*60)
 X_train['uid'] = X_train.card1_addr1.astype(str)+'_'+(X_train.day-X_train.D1 ).floor().astype(str)
 X_test['day'] = X_test.TransactionDT /(24*60*60)
 X_test['uid'] = X_test.card1_addr1.astype(str)+'_'+(X_test.day-X_test.D1 ).floor().astype(str)
 encode_LE(X_train,X_test,'uid',verbose=False )<feature_engineering>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
%%time
add_features(X_train,X_test,['uid'],['TransactionDT'],['count'])
 add_features(X_train,X_test,['uid'],['TransactionAmt','D4','D9','D10','D15'],['mean','std'])
 add_features(X_train,X_test,['uid'],['C'+str(x)for x in range(1,15)if x!=3],['mean'])
 add_features(X_train,X_test,['uid'],['M'+str(x)for x in range(1,10)],['mean'])
 add_features(X_train,X_test,['uid'],['P_emaildomain','dist1','DT_M','id_02','cents'],['nunique'])
 add_features(X_train,X_test,['uid'],['C14'],['std'])
 add_features(X_train,X_test,['uid'],['C13','V314'],['nunique'])
 add_features(X_train,X_test,['uid'],['V127','V136','V309','V307','V320'],['nunique'] )<drop_column>
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )
cols = list(X_train.columns)
 cols.remove('TransactionDT'); cols.remove('isFraud')
 for c in ['D6','D7','D8','D9','D12','D13','D14'] + ['DT_M','day','uid']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )<train_model>
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
print('NOW USING THE FOLLOWING',len(cols),'FEATURES.')
 print("We read data from disk and created all these features in %.2f seconds!" %(time.time() -start))
np.array(cols )<sort_values>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint1.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
X_train = X_train.sort_index()
split = 3*len(X_train)//4<prepare_x_and_y>
class TrainGlobalConfig:
num_workers = 2
batch_size = 4
n_epochs = 14
lr = 0.0004
folder = 'plabel_model'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)
if LOCAL_VALIDATION:
print("XGBoost version:", xgb.__version__)
 xgb_parms = {
'n_estimators':2000,
'max_depth':12,
'learning_rate':0.02,
'subsample':0.8,
'colsample_bytree':0.4,
'missing':-1,
'eval_metric':'auc',
'objective':'binary:logistic',
'tree_method':'gpu_hist'
}
train = xgb.DMatrix(data=X_train.iloc[:split][cols],label=X_train.iloc[:split]['isFraud'])
 valid = xgb.DMatrix(data=X_train.iloc[split:][cols],label=X_train.iloc[split:]['isFraud'])
 clf = xgb.train(xgb_parms, dtrain=train,
num_boost_round=2000,evals=[(train,'train'),(valid,'valid')],
early_stopping_rounds=100,maximize=True,
verbose_eval=50 )<save_to_csv>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train = X_train.reset_index()
X_train[['TransactionID','oof']].to_pandas().to_csv('oof_xgb_96.csv')
 X_train = X_train.set_index('TransactionID',drop=True )<save_to_csv>
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_96.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB96 Submission')
 plt.show()<import_modules>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('/kaggle/input/google-weight/last-checkpoint.bin' )
import numpy as np
import pandas as pd
import os
import gc
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import GroupKFold
import xgboost as xgb<define_variables>
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
str_type = ['ProductCD', 'card4', 'card6', 'P_emaildomain', 'R_emaildomain','M1', 'M2', 'M3', 'M4','M5',
'M6', 'M7', 'M8', 'M9', 'id_12', 'id_15', 'id_16', 'id_23', 'id_27', 'id_28', 'id_29', 'id_30',
'id_31', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
cols = ['TransactionID', 'TransactionDT', 'TransactionAmt',
'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8',
'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4',
'M5', 'M6', 'M7', 'M8', 'M9']
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
cols += ['V'+str(x)for x in v]
dtypes = {}
for c in cols+['id_0'+str(x)for x in range(1,10)]+['id_'+str(x)for x in range(10,34)]:
dtypes[c] = 'float32'
for c in str_type: dtypes[c] = 'object'<load_from_csv>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
%%time
folder_path = '.. /input/ieee-fraud-detection/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID', dtype=dtypes)
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID', dtype=dtypes, usecols=cols+['isFraud'])
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID', dtype=dtypes)
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID', dtype=dtypes, usecols=cols)
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
' )<merge>
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
print('Merging data...')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()<data_type_conversions>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
%%time
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<feature_engineering>
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
image_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p )
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 train['DT_M'] = train['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train['DT_M'] =(train['DT_M'].dt.year-2017)*12 + train['DT_M'].dt.month
test['DT_M'] = test['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 test['DT_M'] =(test['DT_M'].dt.year-2017)*12 + test['DT_M'].dt.month<drop_column>
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df.head()
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
results_df['image_id'] = results_df['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg' )
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()<init_hyperparams>
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_ROOT_PATH+'/'+ x+'.jpg' )
xgb_params = {'n_estimators':5000,
'max_depth':12,
'learning_rate':0.02,
'subsample':0.8,
'colsample_bytree':0.4,
'missing':-1,
'eval_metric':'auc',
'tree_method':'gpu_hist'
}<train_on_grid>
train_data_plabel = pd.concat([results_df,marking], axis=0 )
%%time
oof = np.zeros(len(X))
preds = np.zeros(len(X_test))
skf = GroupKFold(n_splits=6)
 for i,(idxT, idxV)in enumerate(skf.split(X, y, groups=X['DT_M'])) :
month = X.iloc[idxV]['DT_M'].iloc[0]
print('Fold',i,'withholding month',month)
 print('rows of train =',len(idxT),'rows of holdout =',len(idxV))
clf = xgb.XGBClassifier(**xgb_params)
 h = clf.fit(X[cols].iloc[idxT], y.iloc[idxT],
eval_set=[(X[cols].iloc[idxV],y.iloc[idxV])],
verbose=100, early_stopping_rounds=200)
 oof[idxV] += clf.predict_proba(X[cols].iloc[idxV])[:,1]
preds += clf.predict_proba(X_test[cols])[:,1]/skf.n_splits
del h, clf
x = gc.collect()
print('
print('XGB95 OOF CV=',roc_auc_score(y,oof))<save_to_csv>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
sub['isFraud'] = preds
sub.to_csv("submission_xgb.csv", index=False )<set_options>
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 500 )<load_from_csv>
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds.index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True,
)
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv')
 sub = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<categorify>
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
booleanDictionary = {'T':True, 'F':False}
train = train.replace(booleanDictionary)
 test = test.replace(booleanDictionary )<count_missing_values>
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load('/kaggle/input/google-weight/last-checkpoint.bin')
 net.load_state_dict(checkpoint['model_state_dict'])
 return DetBenchTrain(net, config)
 net = get_net()
train['nulls1'] = train.isna().sum(axis=1)
 test['nulls1'] = test.isna().sum(axis=1 )<feature_engineering>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 for df in [train, test]:
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds=x)))
 df['DT_M'] =(df['DT'].dt.year - 2017)* 12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year - 2017)* 52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year - 2017)* 365 + df['DT'].dt.dayofyear
df['hour'] = df['DT'].dt.hour
df['dow'] = df['DT'].dt.dayofweek
df['day'] = df['DT'].dt.day<categorify>
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
pass
else:
run_training()
def values_normalization(dt_df, periods, columns):
for period in periods:
for col in columns:
new_col = col +'_'+ period
dt_df[col] = dt_df[col].astype(float)
 temp_min = dt_df.groupby([period])[col].agg(['min'] ).reset_index()
temp_min.index = temp_min[period].values
temp_min = temp_min['min'].to_dict()
temp_max = dt_df.groupby([period])[col].agg(['max'] ).reset_index()
temp_max.index = temp_max[period].values
temp_max = temp_max['max'].to_dict()
temp_mean = dt_df.groupby([period])[col].agg(['mean'] ).reset_index()
temp_mean.index = temp_mean[period].values
temp_mean = temp_mean['mean'].to_dict()
temp_std = dt_df.groupby([period])[col].agg(['std'] ).reset_index()
temp_std.index = temp_std[period].values
temp_std = temp_std['std'].to_dict()
dt_df['temp_min'] = dt_df[period].map(temp_min)
 dt_df['temp_max'] = dt_df[period].map(temp_max)
 dt_df['temp_mean'] = dt_df[period].map(temp_mean)
 dt_df['temp_std'] = dt_df[period].map(temp_std)
 dt_df[new_col+'_min_max'] =(dt_df[col]-dt_df['temp_min'])/(dt_df['temp_max']-dt_df['temp_min'])
 dt_df[new_col+'_std_score'] =(dt_df[col]-dt_df['temp_mean'])/(dt_df['temp_std'])
 del dt_df['temp_min'],dt_df['temp_max'],dt_df['temp_mean'],dt_df['temp_std']
return dt_df
i_cols = ['D1','D2','D3','D4','D5','D6','D7','D8','D9','D10','D11','D12','D13','D14','D15']
for col in i_cols:
df = pd.concat([train[[col,'DT_M']],test[[col,'DT_M']]])
 df = values_normalization(df,['DT_M'],[col])
 print(df.head())
 train = pd.concat([train,df.iloc[:len(train),len(df.columns)-2:]],axis=1)
 test = pd.concat([test,df.iloc[len(train):,len(df.columns)-2:]],axis=1)
 del df
gc.collect()<feature_engineering>
weights = f'./plabel_model/last-checkpoint1.bin'
if not os.path.exists(weights):
weights = '/kaggle/input/google-weight/last-checkpoint.bin'
net = load_net(weights )
def setbrowser(df):
df.loc[df["id_31"]=="samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"]=="opera 53.0",'lastest_browser']=1
df.loc[df["id_31"]=="mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"]=="google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"]=="firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"]=="edge 17.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for ios",'lastest_browser']=1
return df
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 train = setbrowser(train)
 test = setbrowser(test )<sort_values>
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )
i_cols = ['TransactionID','card1','card2','card3','card4','card5','card6']
full_df = pd.concat([train[i_cols], test[i_cols]])
 full_df['card6'] = np.where(full_df['card6']==30, np.nan, full_df['card6'])
 full_df['card6'] = np.where(full_df['card6']==16, np.nan, full_df['card6'])
 i_cols = ['card2','card3','card4','card5','card6']
for col in i_cols:
temp_df = full_df.groupby(['card1',col])[col].agg(['count'] ).reset_index()
temp_df = temp_df.sort_values(by=['card1','count'], ascending=False ).reset_index(drop=True)
 del temp_df['count']
temp_df = temp_df.drop_duplicates(keep='first' ).reset_index(drop=True)
 temp_df.index = temp_df['card1'].values
temp_df = temp_df[col].to_dict()
full_df[col] = np.where(full_df[col].isna() , full_df['card1'].map(temp_df), full_df[col])
 i_cols = ['card1','card2','card3','card4','card5','card6']
for col in i_cols:
train[col] = full_df[full_df['TransactionID'].isin(train['TransactionID'])][col].values
test[col] = full_df[full_df['TransactionID'].isin(test['TransactionID'])][col].values<set_options>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
weights = f'./plabel_model/last-checkpoint1.bin'
if not os.path.exists(weights):
weights = '/kaggle/input/google-weight/last-checkpoint.bin'
models = [
load_net(weights)
 ]
del full_df
gc.collect()<define_variables>
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes
card1_thred = 2<feature_engineering>
all_predictions = []
for fold_number in [0]:
validation_dataset = DatasetRetriever(
 image_ids=np.array([i for i in df_folds[df_folds['fold'] == fold_number].index.values if 'train' in i]),
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 for images, targets, image_ids in tqdm(val_loader, total=len(val_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
det = models[0](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
all_predictions.append({
'pred_boxes':(boxes*2 ).clip(min=0, max=1023 ).astype(int),
'scores': scores,
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
'image_id': image_ids[i],
} )
valid_card = train['card1'].value_counts()
valid_card = valid_card[valid_card>card1_thred]
valid_card = list(valid_card.index)
 train['card1'] = np.where(train['card1'].isin(valid_card), train['card1'], np.nan)
 test['card1'] = np.where(test['card1'].isin(valid_card), test['card1'], np.nan )<count_values>
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )
train.loc[train.card3.isin(train.card3.value_counts() [train.card3.value_counts() < 200].index), 'card3'] = "Others"
test.loc[test.card3.isin(test.card3.value_counts() [test.card3.value_counts() < 200].index), 'card3'] = "Others"
train.loc[train.card5.isin(train.card5.value_counts() [train.card5.value_counts() < 300].index), 'card5'] = "Others"
test.loc[test.card5.isin(test.card5.value_counts() [test.card5.value_counts() < 300].index), 'card5'] = "Others"<data_type_conversions>
best_final_score, best_score_threshold = 0, 0
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
train['uid'] = train['card1'].astype(str)+'_'+train['card2'].astype(str)
 test['uid'] = test['card1'].astype(str)+'_'+test['card2'].astype(str)
 train['uid2'] = train['uid'].astype(str)+'_'+train['card3'].astype(str)+'_'+train['card5'].astype(str)
 test['uid2'] = test['uid'].astype(str)+'_'+test['card3'].astype(str)+'_'+test['card5'].astype(str)
 train['uid3'] = train['uid2'].astype(str)+'_'+train['addr1'].astype(str)+'_'+train['addr2'].astype(str)
 test['uid3'] = test['uid2'].astype(str)+'_'+test['addr1'].astype(str)+'_'+test['addr2'].astype(str)
 train['uid4'] = train['uid3'].astype(str)+'_'+train['P_emaildomain'].astype(str)
 test['uid4'] = test['uid3'].astype(str)+'_'+test['P_emaildomain'].astype(str)
 train['uid5'] = train['uid3'].astype(str)+'_'+train['R_emaildomain'].astype(str)
 test['uid5'] = test['uid3'].astype(str)+'_'+test['R_emaildomain'].astype(str)
<data_type_conversions>
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30 )
train['bank_type'] = train['card3'].astype(str)+'_'+train['card5'].astype(str)
 test['bank_type'] = test['card3'].astype(str)+'_'+test['card5'].astype(str)
 train['address_match'] = train['bank_type'].astype(str)+'_'+train['addr2'].astype(str)
 test['address_match'] = test['bank_type'].astype(str)+'_'+test['addr2'].astype(str)
 for col in ['address_match','bank_type']:
tmp = pd.concat([train[[col]], test[[col]]])
 tmp[col] = np.where(tmp[col].str.contains('nan'), np.nan, tmp[col])
 tmp = tmp.dropna()
fq_encode = tmp[col].value_counts().to_dict()
train[col] = train[col].map(fq_encode)
 test[col] = test[col].map(fq_encode)
 train['address_match'] = train['address_match']/train['bank_type']
test['address_match'] = test['address_match']/test['bank_type']<feature_engineering>
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indexes = np.where(scores>best_score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
train['D9_na'] = np.where(train['D9'].isna() , 0, 1)
 test['D9_na'] = np.where(test['D9'].isna() , 0, 1)
 train['local_hour'] = train['D9']*24
test['local_hour'] = test['D9']*24
train['local_hour'] = train['local_hour'] -(train['TransactionDT']/(60*60)) %24
test['local_hour'] = test['local_hour'] -(test['TransactionDT']/(60*60)) %24
train['local_hour_dist'] = train['local_hour']/train['dist2']
test['local_hour_dist'] = test['local_hour']/test['dist2']<data_type_conversions>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
train['M_sum'] = train[i_cols].sum(axis=1 ).astype(np.int8)
 test['M_sum'] = test[i_cols].sum(axis=1 ).astype(np.int8)
 train['M_na'] = train[i_cols].isna().sum(axis=1 ).astype(np.int8)
 test['M_na'] = test[i_cols].isna().sum(axis=1 ).astype(np.int8 )<data_type_conversions>
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
for j,(images, image_ids)in enumerate(data_loader):
predictions = make_tta_predictions(images)
 i = 0
sample = images[i].permute(1,2,0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=511)
 indexes = np.where(scores>best_score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for box, score in zip(boxes,scores):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(1, 0, 0), 1)
 cv2.putText(sample, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample);