Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<create_dataframe>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
def parse_sample(sample : pd.DataFrame,
batch_no : int,
batch_size : int,
window_size : List)-> pd.DataFrame:
temp = feature_enginering_by_batch(sample['signal'].values, batch_size, window_size)
 temp['batch_'+ str(batch_size)] = int(batch_no)
 return temp 
<define_variables>
%%time
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = False
models = []
for w in weights:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
for model in models:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, 0.4, opt.iou_thres,merge=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
class opt:
weights = ['.. /input/foldogw/best.pt','.. /input/fold1gwnew/best.pt','.. /input/fold2gwnew//best.pt',
'.. /input/yolofold330epochs/weights/best_yolov5x_fold3.pt',
'.. /input/fold3wheat30/weights/best_yolov5x_fold4.pt']
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()
def sample_gen(df : pd.DataFrame,
batch_size : int,
window_size : List,
batches : List=[0],)-> pd.DataFrame:
result = Parallel(n_jobs=1, temp_folder='/tmp', max_nbytes=None, backend='multiprocessing' )(delayed(parse_sample)
(df[df['batch_'+ str(batch_size)]==i], int(i), batch_size, window_size)
 for i in tqdm(batches))
data = [r.values for r in result]
data = np.vstack(data)
 cols = result[0].columns
cols = [name+'_'+str(batch_size)if name!='batch_'+ str(batch_size)else 'batch_'+ str(batch_size)for name in cols ]
X = pd.DataFrame(data, columns=cols)
 X = reduce_mem_usage(X, False)
 X = X.sort_values('batch_'+ str(batch_size))
return X 
<create_dataframe>
def run_wbf_yolo(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.34, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
add_index : bool,
batch_size : int,
window_size : List)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, add_index=add_index)
 if create_all_data_feats:
df = lag_with_pct_change(df, batch_size, [1, 2, 4], add_pct_change=True, add_pct_change_lag=True)
 batches = df['batch_'+ str(batch_size)].unique().tolist()
batch_feats=sample_gen(df, batch_size=batch_size, window_size=window_size, batches=batches)
 df = pd.merge(df, batch_feats, on='batch_'+ str(batch_size), how='left')
 df = reduce_mem_usage(df, False)
 return df 
<create_dataframe>
all_path,all_score,all_bboxex = res
yolov5preds = {}
for row in range(len(all_path)) :
preds = {}
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf_yolo(boxes,scores)
 yolov5preds[image_id] = [boxes,scores,labels]
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame,
subtract_only : Optional[bool]=True,
idx_cols : List=['time'],
target_col : List=['open_channels'])-> Tuple[pd.DataFrame , pd.DataFrame]:
drops = df.columns[df.isna().sum() >25000]
df = df.drop(drops, axis=1)
 df = df.replace([np.inf, -np.inf], np.nan)
 df = df.fillna(0)
 gc.collect()
if subtract_only == False:
corrcoef_cols = [col for col in df.columns.tolist() if col not in(idx_cols+target_col)]
first=dict() ; second=dict() ; third=dict()
for col in corrcoef_cols:
ss = np.corrcoef(df[col], df['open_channels'])[0, 1]
first[col] = ss
ss = np.corrcoef(df[col]-df['signal'], df['open_channels'])[0, 1]
second[col] = ss
ss = np.corrcoef(df[col]*df['signal'], df['open_channels'])[0, 1]
third[col] = ss
corr_df = pd.DataFrame.from_dict(
 {
'Base':first,
'Signal-Subtracted': second,
'Signal-Multiplied': third
}
).fillna(0 ).apply(np.abs ).sort_values('Base', ascending=False)
 base_cols = corr_df.sort_values('Base', ascending=False ).head(100 ).index.tolist()
multiply_cols = corr_df.sort_values('Signal-Multiplied', ascending=False ).head(10 ).index.tolist()
subtract_cols = corr_df.sort_values('Signal-Subtracted', ascending=False ).head(25 ).index.tolist()
display(corr_df.sort_values('Base', ascending=False ).tail(50))
all_cols = list(set(base_cols + multiply_cols + subtract_cols + idx_cols + target_col))
all_cols_test = list(set(base_cols + multiply_cols + subtract_cols + idx_cols))
drops = list(set(multiply_cols + subtract_cols)-set(base_cols))
df = df[all_cols]
df_test = df_test[all_cols_test]
for col in multiply_cols:
df[col+'_m'] = df[col] * df['signal']
df_test[col+'_m'] = df_test[col] * df_test['signal']
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = df.drop(drops, axis=1)
 else:
not_imp = ['kstat_1_5000', 'kstat_2_5000', 'kstat_3_5000', 'kstat_4_5000', 'moment_2_5000',
'moment_3_5000','moment_4_5000', 'kstatvar_1_5000', 'kstatvar_2_5000','kstat_1_50000',
'kstat_2_50000', 'kstat_3_50000', 'kstat_4_50000', 'moment_2_50000', 'moment_3_50000',
'moment_4_50000', 'kstatvar_1_50000', 'kstatvar_2_50000']
subtract_cols = list(set(df.columns.tolist())-set(idx_cols + target_col + not_imp))
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = reduce_mem_usage(df, False)
 df_test = reduce_mem_usage(df_test, False)
 gc.collect()
return df, df_test 
<compute_test_metric>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
def MacroF1Metric(preds : np.array, dtrain : lgb.Dataset)-> Tuple[Text, np.array, bool] :
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
<create_dataframe>
best_iou_thr = 0.432
best_skip_box_thr = 0.397
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
shuffle : bool,
seed : int,
batch_col : Text,
params : Dict,
feats : List,
sample_submission: pd.DataFrame)-> pd.DataFrame:
oof_ = np.zeros(len(train))
preds_ = np.zeros(len(test))
target = ['open_channels']
imp_df = pd.DataFrame(index=feats)
 kf = KFold(splits, shuffle, seed)
 for n_fold,(tr_idx, val_idx)in enumerate(kf.split(train, train[target], groups=train[batch_col])) :
tr_x = train[feats].iloc[tr_idx]
vl_x = train[feats].iloc[val_idx]
tr_y = train[target].iloc[tr_idx].values
vl_y = train[target].iloc[val_idx].values
model = HistGradientBoostingRegressor(learning_rate = 0.1, max_iter=800, random_state = 404, validation_fraction=None, verbose = 0, max_depth=12, min_samples_leaf=25, l2_regularization=0.05)
 model.fit(tr_x, tr_y)
 oof_[val_idx] += model.predict(train[feats].iloc[val_idx])
 preds_ += model.predict(test[feats])/ SPLITS
f1_score_ = f1_score(train[target].iloc[val_idx], np.round(np.clip(oof_[val_idx], 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target].iloc[val_idx], oof_[val_idx]))
logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f} rmse score : {rmse_score_:1.5f}')
 f1_score_ = f1_score(train[target], np.round(np.clip(oof_, 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target], oof_))
logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f} oof rmse score : {rmse_score_:1.5f}')
 sample_submission['open_channels'] = np.round(np.clip(preds_, 0, 10)).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return imp_df 
<init_hyperparams>
models = [
load_net('.. /input/kernel5c5dc38533/effdet5-cutmix-augmix-bboxaug-0/last-checkpoint.bin'),
load_net('.. /input/fold-1-global-wheat/effdet5-cutmix-augmix-bboxaug-1/last-checkpoint.bin'),
load_net('.. /input/fold-2-global-wheat/effdet5-cutmix-augmix-bboxaug-2/last-checkpoint.bin'),
load_net('.. /input/fold-3-bbox-aug-50/effdet5-cutmix-augmix-bboxaug-3/last-checkpoint.bin'),
load_net('.. /input/fold-4-bbox-aug-50/effdet5-cutmix-augmix-bboxaug-4/last-checkpoint.bin'),
load_net('.. /input/fold-0/effdet5-cutmix-augmix-0/last-checkpoint.bin'),
load_net('.. /input/fold-1/effdet5-cutmix-augmix-1/last-checkpoint.bin'),
load_net('.. /input/fold-2-augmix/effdet5-cutmix-augmix-2/last-checkpoint.bin'),
load_net('.. /input/fold-4/effdet5-cutmix-augmix-4/last-checkpoint.bin')
 ]
def get_params(seed : int)-> Dict :
params = dict()
params['learning_rate']=0.009;
params['max_depth']=-1;
params['num_leaves']=257;
params['metric']='rmse';
params['random_state']=seed;
params['n_jobs']=-1;
params['feature_fraction']=1 ;
params['boosting']='goss';
params['boost_from_average']=True;
params['bagging_seed']=seed;
params['bagging_freq']=0;
params['bagging_fraction']=1;
params['reg_alpha']=0;
params['reg_lambda']=0
params['force_row_wise']=True
return params<load_pretrained>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob.glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(
 images,
score_threshold=0.25, 
):
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 logger.info('Reading Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], add_index=config[1], batch_size=config[2], window_size=config[3])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], add_index=config[1], batch_size=config[2], window_size=config[3])
 not_feats_cols.append('batch_'+str(config[2]))
if config[1]:
not_feats_cols.append('batch_'+str(config[2])+'_idx')
 if SELECT:
train, test = feature_selection(train, test, subtract_only=True, idx_cols=not_feats_cols, target_col=target_col)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running HistGradientBoosting model'):
logger.info(f'Training HistGradientBoosting model with {SPLITS} folds Started...')
 params = get_params(SEED)
 feats = [c for c in train.columns if c not in(not_feats_cols+target_col)]
importances = run_cv_model_by_batch(train, test, splits=SPLITS, shuffle=True, seed=SEED, batch_col='batch_50000',params=params, feats=feats, sample_submission=sample_submission)
 importances.to_csv('importances.csv')
 logger.info(f'Training completed...')
<import_modules>
effdetpreds = {}
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2)
 image_id = image_ids[i]
effdetpreds[image_id] = [boxes,scores,labels]
import os
import gc
import numpy as np
import pandas as pd
import time
import math
from numba import jit
from math import log, floor
import scipy
from scipy import signal
from scipy.signal import butter, deconvolve
from sklearn.neighbors import KDTree
from pathlib import Path
from sklearn.utils import shuffle
import seaborn as sns
from matplotlib import colors
import matplotlib.pyplot as plt
from matplotlib.colors import Normalize
import lightgbm as lgb
from tqdm import tqdm
from sklearn.model_selection import GroupKFold, KFold
from sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error, f1_score<load_from_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
DATA_PATH = ".. /input/liverpool-ion-switching"
train_df = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))
test_df = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))
submission_df = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))<feature_engineering>
y = effdetpreds.copy()
x = yolov5preds.copy()
def get_stats(df):
stats = pd.DataFrame(index=df.columns, columns=['na_count', 'n_unique', 'type', 'memory_usage'])
 for col in df.columns:
stats.loc[col] = [df[col].isna().sum() , df[col].nunique(dropna=False), df[col].dtypes, df[col].memory_usage(deep=True, index=False)/ 1024**2]
stats.loc['Overall'] = [stats['na_count'].sum() , stats['n_unique'].sum() , None, df.memory_usage(deep=True ).sum() / 1024**2]
return stats
def print_header() :
print('col conversion dtype na uniq size')
 print()
def print_values(name, conversion, col):
template = '{:10} {:16} {:>7} {:2} {:6} {:1.2f}MB'
print(template.format(name, conversion, str(col.dtypes), col.isna().sum() , col.nunique(dropna=False), col.memory_usage(deep=True, index=False)/ 1024 ** 2))<compute_test_metric>
def run_last_wbf(yolo,eff,weights=[2,1],iou_thr=0.5,skip_box_thr=0.397):
box1,scores1,labels1 = yolo
box2,scores2,labels2 = eff
box1 = box1/1023
box2 = box2/1023
boxes, scores, labels = weighted_boxes_fusion([box1,box2], [scores1,scores2],
[labels1,labels2],weights=[2,1],
iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes,scores
get_stats(test_df )<normalization>
iou_thr = 0.5
skip_box_thr = 0.0001
results = []
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes,scores = run_last_wbf(yolov5preds[image_id],effdetpreds[image_id])
 boxes =(boxes*1023 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'] )
<feature_engineering><EOS> 
test_df.to_csv('submission.csv', index=False)
 test_df.head()