Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<load_from_csv>
!pip install --no-deps '.. /input/timm0130/timm-0.1.30-py3-none-any.whl' > /dev/null
data_train = pd.read_csv('.. /input/train_users_2.csv',parse_dates=['timestamp_first_active','date_account_created','date_first_booking'])
 data_test = pd.read_csv('.. /input/test_users.csv',parse_dates=['timestamp_first_active','date_account_created','date_first_booking'])
 data_train.head()<load_from_csv>
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
countries = pd.read_csv('.. /input/countries.csv')
 countries.head(10 )<drop_column>
%matplotlib inline
VerticalFlip, HorizontalFlip, IAASharpen,
OneOf, Compose , BboxParams, Resize, HueSaturationValue
,RandomBrightnessContrast, ToGray , Cutout , RandomSizedCrop )
test_ids = data_test['id']
Nrows_train = data_train.shape[0]
labels = data_train['country_destination'].values
data_train1 = data_train.drop(['country_destination'], axis=1)
 data_all = pd.concat(( data_train1, data_test), axis = 0, ignore_index = True)
 data_all = data_all.drop(['id','date_first_booking'], axis=1)
<count_missing_values>
from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain , DetBenchPredict
from effdet.efficientdet import HeadNet
print(data_all.isnull().sum() )<feature_engineering>
DIR_PATH = '/kaggle/input/global-wheat-detection/'
dir = glob.glob(os.path.join(DIR_PATH , '*'))
dir.sort(reverse=True)
 train_paths = glob.glob(os.path.join(dir[1] , '*'))
test_paths = glob.glob(os.path.join(dir[2] , '*'))
data_all.loc[data_all.age > 100, 'age'] = np.nan
data_all.loc[data_all.age < 18, 'age'] = np.nan<feature_engineering>
df = pd.read_csv(dir[0])
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df
data_all['dac_year'] = data_all.date_account_created.dt.year
data_all['dac_month'] = data_all.date_account_created.dt.month
data_all['dac_day'] = data_all.date_account_created.dt.day
data_all['tfa_year'] = data_all.timestamp_first_active.dt.year
data_all['tfa_month'] = data_all.timestamp_first_active.dt.month
data_all['tfa_day'] = data_all.timestamp_first_active.dt.day
data_all.drop('date_account_created',1, inplace=True)
 data_all.drop('timestamp_first_active',1, inplace=True )<groupby>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = df[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = df[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds
data_all.groupby('gender' ).age.agg(['min','max','mean','count'] )<count_values>
image_id = '8425a537b.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image without bounding box" )
data_all.language.value_counts(sort=True )<count_missing_values>
image_id = 'b3c96d5ad.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image with bounding box" )
print(data_all.isnull().sum() )<categorify>
images_after_augmentation(image, boxes, Blur(blur_limit=7 ,p=1))
features = ['gender','signup_method','signup_flow','language','affiliate_channel','affiliate_provider',\
'first_affiliate_tracked','signup_app','first_device_type','first_browser']
data_all = pd.get_dummies(data_all,columns=features)
<categorify>
images_after_augmentation(image, boxes, HorizontalFlip(p=1))
V = data_all.values
X_train = V[:Nrows_train]
X_test = V[Nrows_train:]
labler = LabelEncoder()
y = labler.fit_transform(labels)
 xgb = XGBClassifier(max_depth=6, learning_rate=0.3, n_estimators=22,
objective='multi:softprob', subsample=0.6, colsample_bytree=0.6, seed=0)
 xgb.fit(X_train, y)
 y_pred = xgb.predict_proba(X_test )<save_to_csv>
class WheatDataset(Dataset):
def __init__(self , dataframe , image_ids, transforms = None):
super().__init__()
self.image_ids = image_ids
self.dataframe = dataframe
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['labels'] = torch.stack(sample['labels'])
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[1]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.dataframe[self.dataframe['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
area =(boxes[:, 2] * boxes[:, 3])
 boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
area = torch.as_tensor(area, dtype=torch.float32)
 return image, boxes
ids = []
cts = []
for i in range(len(test_ids)) :
idx = test_ids[i]
ids += [idx] * 5
cts += labler.inverse_transform(np.argsort(y_pred[i])[::-1])[:5].tolist()
sub = pd.DataFrame(np.column_stack(( ids, cts)) , columns=['id', 'country'])
 sub.to_csv('submission.csv',index=False )<install_modules>
def collate_fn(batch):
return tuple(zip(*batch))
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q<feature_engineering>
bs = 2
fold_number = 0
train_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] != fold_number].index.values , transforms=transforms_train)
 valid_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] == fold_number].index.values , transforms=transforms_valid)
 train_loader = DataLoader(train_set,batch_size = bs,sampler=RandomSampler(train_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=4)
 valid_loader = DataLoader(valid_set,batch_size = bs ,sampler=SequentialSampler(valid_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=4)
 images , targets , path_images = next(iter(train_loader))
img = visualizeTarget(images[0],targets[0])
 visualize(Example_one_image_from_dataloader = img )
os.environ['SM_FRAMEWORK'] = 'tf.keras' 
<load_pretrained>
class Training:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.best_calc_loss = 10**5
self.model = model
self.device = device
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 def train_loop(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
print(f'
{timestamp}
LR: {lr}')
 t = time.time()
calc_loss = self.train_one_epoch(train_loader)
 print(f'Train.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save_model(f'{self.base_dir}/last-epoch.bin')
 t = time.time()
calc_loss = self.valid_one_epoch(validation_loader)
 print(f'Val.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if calc_loss.avg < self.best_calc_loss:
self.best_calc_loss = calc_loss.avg
self.model.eval()
self.save_model(f'{self.base_dir}/efficient-best-{str(self.epoch ).zfill(2)}epoch.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=calc_loss.avg)
 self.epoch += 1
def train_one_epoch(self, train_loader):
self.model.train()
calc_loss = Calculator()
t = time.time()
target_res = {}
for images, targets, image_ids in tqdm_notebook(train_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res['bbox'] = boxes
target_res['cls'] = labels
self.optimizer.zero_grad()
outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 loss.backward()
self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return calc_loss
def valid_one_epoch(self, val_loader):
self.model.eval()
calc_loss = Calculator()
t = time.time()
with torch.no_grad() :
for images, targets, image_ids in tqdm_notebook(val_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res = {}
target_res['bbox'] = boxes
target_res['cls'] = labels
target_res["img_scale"] = torch.tensor([1.0] * batch_size, dtype=torch.float ).to(self.device)
 target_res["img_size"] = torch.tensor([images[0].shape[-2:]] * batch_size, dtype=torch.float ).to(self.device)
 outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 return calc_loss
def save_model(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_calc_loss': self.best_calc_loss,
'epoch': self.epoch,
}, path)
 def load_model(self, path):
model = torch.load(path)
 self.model.model.load_state_dict(model['model_state_dict'])
 self.optimizer.load_state_dict(model['optimizer_state_dict'])
 self.scheduler.load_state_dict(model['scheduler_state_dict'])
 self.best_calc_loss = model['best_calc_loss']
self.epoch = model['epoch'] + 1
with open('.. /input/train-fpn-segmentation-model-no-43/hparams.json')as json_file:
hparams = json.load(json_file)
 hparams<normalization>
def get_model(num_classes = 1):
config = get_efficientdet_config('tf_efficientdet_d7x')
 model = EfficientDet(config, pretrained_backbone=False)
 checkpoint = torch.load('.. /input/efficientdetd7x/tf_efficientdet_d7x-f390b87c.pth')
 model.load_state_dict(checkpoint)
 config.num_classes = num_classes
config.image_size = 512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
return DetBenchTrain(model, config )
IMG_SIZE = hparams['IMG_SIZE']
SCALE_FACTOR = hparams['SCALE_FACTOR']
K_SPLITS = hparams['K_SPLITS']
def read_tif_file(fname):
img = io.imread(fname)
 img = np.squeeze(img)
 if img.shape[0] == 3:
img = img.swapaxes(0,1)
 img = img.swapaxes(1,2)
 return img
def map_img2file(fname):
img = read_tif_file(fname)
 dims = np.array(img.shape)
 ch = 1 if len(dims)== 2 else dims[2]
for i in range(ch):
f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='w+', shape=(dims[0], dims[1]))
f[:] = img[:,:,i] if ch > 1 else img[:,:]
del f
return dims
def get_patch_from_file(dims, pos, psize):
ch = 1 if len(dims)== 2 else dims[2]
patch = np.zeros([psize[0], psize[1]], dtype=np.uint8)if ch == 1 else np.zeros([psize[0], psize[1], ch], dtype=np.uint8)
 for i in range(ch):
f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='r', shape=(dims[0], dims[1]))
p = f[pos[0]:pos[0]+psize[0], pos[1]:pos[1]+psize[1]]
crop = p.shape
if ch == 1:
patch[0:p.shape[0], 0:p.shape[1]] = p
else:
patch[0:p.shape[0], 0:p.shape[1],i] = p
del f
return patch, crop<categorify>
class GlobalParametersTrain:
lr = 0.0002
n_epochs = 20
folder = '.. /input/modelfasterrcnn'
verbose = True
verbose_step = 10
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(mode='min',factor=0.5,patience=1,verbose=False, threshold=0.0001,threshold_mode='abs',cooldown=0, min_lr=1e-8,eps=1e-08 )
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<feature_engineering>
def load_model(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d7x')
 model = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchPredict(model, config)
 model.eval() ;
return model.cuda()
model = load_model('.. /input/efficientdet7xbestmodel/efficient-best-11epoch.bin' )
def create_TTA_batch(img):
if len(img.shape)< 4:
img = np.expand_dims(img, 0)
 batch=np.zeros(( img.shape[0]*8,img.shape[1],img.shape[2],img.shape[3]), dtype=np.float32)
 for i in range(img.shape[0]):
orig = tf.keras.preprocessing.image.img_to_array(img[i,:,:,:])/255.
 batch[i*8,:,:,:] = orig
batch[i*8+1,:,:,:] = np.rot90(orig, axes=(0, 1), k=1)
 batch[i*8+2,:,:,:] = np.rot90(orig, axes=(0, 1), k=2)
 batch[i*8+3,:,:,:] = np.rot90(orig, axes=(0, 1), k=3)
 orig = orig[:, ::-1]
batch[i*8+4,:,:,:] = orig
batch[i*8+5,:,:,:] = np.rot90(orig, axes=(0, 1), k=1)
 batch[i*8+6,:,:,:] = np.rot90(orig, axes=(0, 1), k=2)
 batch[i*8+7,:,:,:] = np.rot90(orig, axes=(0, 1), k=3)
 return batch
def mask_TTA(masks):
batch=np.zeros(( masks.shape[0],masks.shape[1],masks.shape[2],masks.shape[3]), dtype=np.float32)
 for i in range(masks.shape[0]//8):
batch[i*8,:,:,:] = masks[i*8]
batch[i*8+1,:,:,:] = np.rot90(masks[i*8+1], axes=(0, 1), k=3)
 batch[i*8+2,:,:,:] = np.rot90(masks[i*8+2], axes=(0, 1), k=2)
 batch[i*8+3,:,:,:] = np.rot90(masks[i*8+3], axes=(0, 1), k=1)
 batch[i*8+4,:,:,:] = masks[i*8+4][:, ::-1]
batch[i*8+5,:,:,:] = np.rot90(masks[i*8+5], axes=(0, 1), k=3)[:, ::-1]
batch[i*8+6,:,:,:] = np.rot90(masks[i*8+6], axes=(0, 1), k=2)[:, ::-1]
batch[i*8+7,:,:,:] = np.rot90(masks[i*8+7], axes=(0, 1), k=1)[:, ::-1]
return(batch )<save_to_csv>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def make_predictions(images , score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
outputs = model(images , torch.tensor([1.0] * images.shape[0], dtype=torch.float ).to(device), torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes],
})
 return [predictions]
MEANING_OF_LIFE = 42
MEANING_OF_LIFE_REV = int(str(MEANING_OF_LIFE)[::-1])
 PATH = '.. /input/hubmap-kidney-segmentation/test/'
filelist = glob.glob(PATH+'*.tiff')
 if len(filelist)== 5:
filelist = filelist[:1]
SUB_FILE = './submission.csv'
with open(SUB_FILE, 'w')as f:
f.write("id,predicted
")
 MODELS = ['.. /input/train-fpn-segmentation-model-no-43/FPN-model-2']
s_th = MEANING_OF_LIFE+K_SPLITS
p_th = IMG_SIZE*IMG_SIZE//32
size = int(IMG_SIZE * SCALE_FACTOR)
 OVERLAP = size//2
STEP = size-OVERLAP
for file in filelist:
fid = file.replace('\','.' ).replace('/','.' ).split('.')[-2]
print(fid)
 dims = map_img2file(file)
 pmask = np.zeros(dims[:2], dtype=np.uint8)
 x_shft, y_shft = 0,0
if fid == 'afa5e8098':
x_shft, y_shft = MEANING_OF_LIFE, MEANING_OF_LIFE_REV
for modl in range(len(MODELS)) :
print(MODELS[modl])
 mname = MODELS[modl]
with open(mname+'.json', 'r')as m:
lm = m.read()
model = model_from_json(lm)
 model.load_weights(mname+'.h5')
 for x in range(( dims[0]-OVERLAP-x_shft)//STEP + min(1,(dims[0]-OVERLAP-x_shft)% STEP)) :
for y in range(( dims[1]-OVERLAP-y_shft)//STEP + min(1,(dims[1]-OVERLAP-y_shft)% STEP)) :
tile, crop = get_patch_from_file(dims, [x*STEP+x_shft, y*STEP+y_shft], [size,size])
 patch = cv2.resize(tile,
dsize=(IMG_SIZE, IMG_SIZE),
interpolation = cv2.INTER_AREA)
 _, s, _ = cv2.split(cv2.cvtColor(patch, cv2.COLOR_BGR2HSV))
if(s>s_th ).sum() > p_th:
batch = create_TTA_batch(patch)
 preds = model.predict(batch)
 pred = mask_TTA(preds)
 mask = np.rint(np.sum(pred, axis=0))
pint =cv2.resize(mask.astype(int), dsize=(size, size), interpolation = cv2.INTER_NEAREST)
 pmask[x*STEP:x*STEP+crop[0], y*STEP:y*STEP+crop[1]] += pint[0:crop[0], 0:crop[1]].astype(np.uint8)
 pmask = pmask >= MEANING_OF_LIFE_REV - K_SPLITS
with open(SUB_FILE, 'a')as f:
f.write("{},".format(fid))
f.write(rle_encode_less_memory(pmask))
f.write("
" )<set_options>
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
%rm -f *.dat<load_from_csv>
for j,(images, targets , image_ids)in enumerate(valid_loader):
break
predictions = make_predictions(images)
 i = 0
sample = images[i].permute(1,2,0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=511 )
df_seeds = pd.read_csv('.. /input/WNCAATourneySeeds_SampleTourney2018.csv')
 df_tour = pd.read_csv('.. /input/WRegularSeasonCompactResults_PrelimData2018.csv' )<merge>
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
df_seeds['seed_int'] = df_seeds['Seed'].apply(lambda x : int(x[1:3]))
df_winseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_dummy = pd.merge(left=df_tour, right=df_winseeds, how='left', on=['Season', 'WTeamID'])
 df_concat = pd.merge(left=df_dummy, right=df_lossseeds, on=['Season', 'LTeamID'] )<feature_engineering>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None], [TTAVerticalFlip() , None],[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
df_concat['DiffSeed'] = df_concat[['LSeed', 'WSeed']].apply(lambda x : 0 if x[0] == x[1] else 1, axis = 1 )<load_from_csv>
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda() , torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
'labels': labels[indexes],
})
 predictions.append(result)
 return predictions
df_sample_sub = pd.read_csv('.. /input/WSampleSubmissionStage2.csv' )<feature_engineering>
test_transforms = Compose([
Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
df_sample_sub['Season'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[0]))
df_sample_sub['TeamID1'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[1]))
df_sample_sub['TeamID2'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[2]))<drop_column>
class TestDataset(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[2]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self):
return self.image_ids.shape[0]
winners = df_concat.rename(columns = { 'WTeamID' : 'TeamID1',
'LTeamID' : 'TeamID2',
'WScore' : 'Team1_Score',
'LScore' : 'Team2_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 winners['Result'] = 1.0
losers = df_concat.rename(columns = { 'WTeamID' : 'TeamID2',
'LTeamID' : 'TeamID1',
'WScore' : 'Team2_Score',
'LScore' : 'Team1_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 losers['Result'] = 0.0
train = pd.concat([winners, losers], axis = 0 ).reset_index(drop = True)
 train['Score_Ratio'] = train['Team1_Score'] / train['Team2_Score']
train['Score_Total'] = train['Team1_Score'] + train['Team2_Score']
train['Score_Pct'] = train['Team1_Score'] / train['Score_Total']<merge>
def collate_fn(batch):
return tuple(zip(*batch))
train_test_inner = pd.merge(train.loc[ train['Season'].isin([2018]), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'inner' )<feature_engineering>
test_set = TestDataset(image_ids=np.array([path.split('/')[-1][:-4] for path in test_paths]),transforms=test_transforms)
 test_loader = DataLoader(test_set,batch_size=4,shuffle=False,num_workers=2,drop_last=False,collate_fn=collate_fn )
team1d_score_spread = train_test_inner.groupby(['Season', 'TeamID1'])[['Score_Ratio', 'Score_Pct']].median().reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio1', 'Score_Pct' : 'Score_Pct1'})
 team2d_score_spread = train_test_inner.groupby(['Season', 'TeamID2'])[['Score_Ratio', 'Score_Pct']].median().reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio2', 'Score_Pct' : 'Score_Pct2'})
 score_spread = team1d_score_spread.join(team2d_score_spread ).reset_index()
score_spread['Score_Ratio'] = score_spread[['Score_Ratio1', 'Score_Ratio2']].apply(lambda x :(x[0] *(x[1] ** -1.0)) , axis = 1)** 0.5
score_spread['Score_Pct'] = score_spread[['Score_Pct1', 'Score_Pct2']].apply(lambda x : 0.5*(x[0] ** -1.0)+ 0.5*(1.0 - x[1])** -1.0, axis = 1)** -1.0
score_spread.head()<merge>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
X_train = train_test_inner.loc[:, ['Season', 'NumOT', 'Score_Ratio', 'Score_Pct']]
train_labels = train_test_inner['Result']
train_test_outer = pd.merge(train.loc[ train['Season'].isin([2014, 2015, 2016, 2017]), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'outer')
 train_test_outer = train_test_outer.loc[ train_test_outer['Result'].isnull() ,
['TeamID1', 'TeamID2', 'Season']]
train_test_missing = pd.merge(pd.merge(score_spread.loc[:, ['TeamID1', 'TeamID2', 'Season', 'Score_Ratio', 'Score_Pct']],
train_test_outer, on = ['TeamID1', 'TeamID2', 'Season']),
num_ot.loc[:, ['TeamID1', 'TeamID2', 'Season', 'NumOT']],
on = ['TeamID1', 'TeamID2', 'Season'] )<categorify>
SUBMISSION_PATH = '/kaggle/working'
submission_id = 'submission'
submission_path = os.path.join(SUBMISSION_PATH, '{}.csv'.format(submission_id))
sample_submission = pd.DataFrame(submission, columns=["image_id","PredictionString"])
 sample_submission.to_csv(submission_path, index=False)
 submission_df = pd.read_csv(submission_path)
 submission_df