Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<feature_engineering>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
def production_countries_preprocessing(elem):
string = str(elem)
 str1 = string.replace(']','' ).replace('[','' ).replace('{','' ).replace('}','' ).replace(' ','' ).replace("name", "" ).replace("iso_3166_1", "" ).replace(":", "" ).replace("'", "")
 ll = str1.split(",")[0::2]
return ll
train_df["production_countries_processed"] = train_df.production_countries.fillna("NaN" ).apply(lambda elem: production_countries_preprocessing(elem))
test_df["production_countries_processed"] = test_df.production_countries.fillna("NaN" ).apply(lambda elem: production_countries_preprocessing(elem))
production_countries_dict = dict()
for production_country in train_df["production_countries_processed"]:
for elem in production_country:
if elem not in production_countries_dict:
production_countries_dict[elem] = 1
else:
production_countries_dict[elem] += 1
production_countries_df = pd.DataFrame.from_dict(production_countries_dict, orient='index')
 production_countries_df.columns = ["number_of_movies"]
production_countries_df = production_countries_df.sort_values(by="number_of_movies", ascending=False)
 production_countries_df.head(10 ).plot.bar()
plt.title("Number of films per production country" )<feature_engineering>
USE_OPTIMIZE = len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) == 10
USE_OPTIMIZE
for c in production_countries_df.index.values:
train_df['isProductionCountry_' + c] = train_df['production_countries_processed'].apply(lambda x: 1 if c in x else 0)
 test_df['isProductionCountry_' + c] = test_df['production_countries_processed'].apply(lambda x: 1 if c in x else 0 )<feature_engineering>
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )
train_df["release_date"] = train_df["release_date"].fillna(train_df["release_date"].mode() [0])
 test_df["release_date"] = test_df["release_date"].fillna(test_df["release_date"].mode() [0])
 train_df['temp'] = train_df.release_date.apply(lambda x: datetime.strptime(x, '%m/%d/%y'))
train_df["month"] = train_df.temp.apply(lambda x: x.month)
 train_df["season"] = train_df["month"]%4
train_df["year"] = train_df.temp.apply(lambda x: x.year)
 train_df["day_of_week"] = train_df.temp.apply(lambda x: x.weekday() +1)
 train_df["week_of_year"] = train_df.temp.apply(lambda x: x.isocalendar() [1])
 train_df = train_df.drop(['temp'], axis=1)
 test_df['temp'] = test_df.release_date.apply(lambda x: datetime.strptime(x, '%m/%d/%y'))
test_df["month"] = test_df.temp.apply(lambda x: x.month)
 test_df["season"] = test_df["month"]%4
test_df["year"] = test_df.temp.apply(lambda x: x.year)
 test_df["day_of_week"] = test_df.temp.apply(lambda x: x.weekday() +1)
 test_df["week_of_year"] = test_df.temp.apply(lambda x: x.isocalendar() [1])
 test_df = test_df.drop(['temp'], axis=1)
 train_df["day_of_week"] = train_df["day_of_week"].fillna(train_df["day_of_week"].mode() [0])
 test_df["day_of_week"] = test_df["day_of_week"].fillna(test_df["day_of_week"].mode() [0])
 train_df["year"] = train_df["year"].fillna(train_df["year"].mode() [0])
 test_df["year"] = test_df["year"].fillna(test_df["year"].mode() [0])
 train_df["month"] = train_df["month"].fillna(train_df["month"].mode() [0])
 test_df["month"] = test_df["month"].fillna(test_df["month"].mode() [0])
 train_df["week_of_year"] = train_df["week_of_year"].fillna(train_df["week_of_year"].mode() [0])
 test_df["week_of_year"] = test_df["week_of_year"].fillna(test_df["week_of_year"].mode() [0])
 train_df["season"] = train_df["season"].fillna(train_df["season"].mode() [0])
 test_df["season"] = test_df["season"].fillna(test_df["season"].mode() [0])
 train_df[["release_date", "month", "year", "day_of_week", "week_of_year", "season"]].head()<define_variables>
%%time
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']]
df_folds['bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds['source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds['stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))
df_holdout = df_folds.iloc[holdout_index].copy()
df_folds = df_folds.iloc[train_index].copy()
df_folds['fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
actors_dict = {}
size_of_actors = len(train_df)- train_df.cast.isna().sum()
for element in train_df[["revenue", "cast"]].values:
if type(element[1])== type(str()):
result = re.findall('name': '\w+\s*\w*', element[1])
 result = [x.replace("name': '", "")for x in result]
for actor in result:
if actor not in actors_dict:
actors_dict[actor] = element[0]
else:
actors_dict[actor] += element[0]
for actor in actors_dict:
actors_dict[actor] = actors_dict[actor]/size_of_actors
actors_df = pd.DataFrame.from_dict(actors_dict, orient='index', columns=["mean_movies_revenue"])
 actors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(20 ).plot.bar()<feature_engineering>
holdout_dataset = DatasetRetriever(
 image_ids=df_holdout.index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 holdout_loader = DataLoader(
 holdout_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)
def find_top_actor_from_cast(top_actor, element):
result = []
if type(element)== type(str()):
result = re.findall('name': '\w+\s*\w*', element)
 result = [x.replace("name': '", "")for x in result]
if top_actor in result:
return 1
else:
return 0
for top_actor in actors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_actor_"+ top_actor] = train_df.cast.apply(lambda element: find_top_actor_from_cast(top_actor, element))
test_df["has_top_actor_"+ top_actor] = test_df.cast.apply(lambda element: find_top_actor_from_cast(top_actor, element))<feature_engineering>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
if USE_OPTIMIZE:
models = [
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold0-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold1-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold2-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold3-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold4-best.bin'),
]
keywords_dict = {}
size_of_keywords = len(train_df)- train_df.Keywords.isna().sum()
for element in train_df[["revenue", "Keywords"]].values:
if type(element[1])== type(str()):
result = re.findall('name': '\w+\s*\w*', element[1])
 result = [x.replace("name': '", "")for x in result]
for key in result:
if key not in keywords_dict:
keywords_dict[key] = element[0]
else:
keywords_dict[key] += element[0]
for key in keywords_dict:
keywords_dict[key] = keywords_dict[key]/size_of_keywords
keywords_df = pd.DataFrame.from_dict(keywords_dict, orient='index', columns=["mean_movies_revenue"])
 keywords_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()<feature_engineering>
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
if USE_OPTIMIZE:
all_predictions = []
for images, targets, image_ids in tqdm(holdout_loader, total=len(holdout_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
fold_predictions = {}
for fold_number in range(5):
fold_predictions[fold_number] = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
image_predictions = {
'image_id': image_ids[i],
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
}
for fold_number in range(5):
boxes, scores = process_det(i, fold_predictions[fold_number])
 image_predictions[f'pred_boxes_fold{fold_number}'] = boxes
image_predictions[f'scores_fold{fold_number}'] = scores
all_predictions.append(image_predictions )
def find_top_keywords_from_cast(top_keyword, element):
result = []
if type(element)== type(str()):
result = re.findall('name': '\w+\s*\w*', element)
 result = [x.replace("name': '", "")for x in result]
if top_keyword in result:
return 1
else:
return 0
for top_keyword in keywords_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_keyword_"+ top_keyword] = train_df.Keywords.apply(lambda element: find_top_keywords_from_cast(top_keyword, element))
test_df["has_top_keyword_"+ top_keyword] = test_df.Keywords.apply(lambda element: find_top_keywords_from_cast(top_keyword, element))<feature_engineering>
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )
directors_dict = {}
size_of_crew = len(train_df)- train_df.crew.isna().sum()
for element in train_df[["revenue", "crew"]].values:
if type(element[1])== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element[1])
 result = [x.replace("Director', 'name': '", "")for x in result]
for key in result:
if key not in directors_dict:
directors_dict[key] = element[0]
else:
directors_dict[key] += element[0]
for key in directors_dict:
directors_dict[key] = directors_dict[key]/size_of_crew
directors_df = pd.DataFrame.from_dict(directors_dict, orient='index', columns=["mean_movies_revenue"])
 directors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()<feature_engineering>
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr,
method,
sigma=0.5, 
):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for fold_number in range(5):
pred_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy()
scores = all_predictions[i][f'scores_fold{fold_number}'].copy()
folds_boxes.append(pred_boxes)
 folds_scores.append(scores)
 folds_labels.append(np.ones(pred_boxes.shape[0]))
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )
def find_top_directors_from_crew(top_director, element):
result = []
if type(element)== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element)
 result = [x.replace("Director', 'name': '", "")for x in result]
if top_director in result:
return 1
else:
return 0
for top_director in directors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_director_"+ top_director] = train_df.crew.apply(lambda element: find_top_directors_from_crew(top_director, element))
test_df["has_top_director_"+ top_director] = test_df.crew.apply(lambda element: find_top_directors_from_crew(top_director, element))<feature_engineering>
%%time
if USE_OPTIMIZE:
print('[WBF]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='weighted_boxes_fusion', 
))
print('[NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='nms', 
))
print('[SOFT NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
sigma=0.1,
method='soft_nms', 
))
print('[NMW]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='non_maximum_weighted', 
))
producers_dict = {}
size_of_crew = len(train_df)- train_df.crew.isna().sum()
for element in train_df[["revenue", "crew"]].values:
if type(element[1])== type(str()):
result = re.findall('Producer', 'name': '\w+\s*\w*', element[1])
 result = [x.replace("Producer', 'name': '", "")for x in result]
for key in result:
if key not in producers_dict:
producers_dict[key] = element[0]
else:
producers_dict[key] += element[0]
for key in producers_dict:
producers_dict[key] = producers_dict[key]/size_of_crew
producers_df = pd.DataFrame.from_dict(producers_dict, orient='index', columns=["mean_movies_revenue"])
 producers_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()<feature_engineering>
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + f'{method}' + '-'*5)
 log(params)
 final_score = calculate_final_score(all_predictions, method=method, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )
def find_top_producers_from_crew(top_producer, element):
result = []
if type(element)== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element)
 result = [x.replace("Director', 'name': '", "")for x in result]
if top_producer in result:
return 1
else:
return 0
for top_producer in producers_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_producer_"+ top_producer] = train_df.crew.apply(lambda element: find_top_producers_from_crew(top_producer, element))
test_df["has_top_producer_"+ top_producer] = test_df.crew.apply(lambda element: find_top_producers_from_crew(top_producer, element))<feature_engineering>
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='nms',
n_calls=50,
)
for i, g in enumerate(genres_df.index.values):
genres_df.loc[g, "median_salary"] = train_df[train_df['isGenre_' + g]==1].revenue.median()
genres_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 )<feature_engineering>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
print('-'*13 + 'NMS' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )
for i, p in enumerate(genres_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopGenre_' + p] = train_df['genres_processed'].apply(lambda x: 1 if p in x else 0)
 train_df['isTopGenre_Other'] = train_df['genres_processed'].apply(lambda x: 1 if p not in x else 0)
 test_df['isTopGenre_' + p] = test_df['genres_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopGenre_Other'] = test_df['genres_processed'].apply(lambda x: 1 if p not in x else 0 )<feature_engineering>
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
Real(0.0, 1, name='sigma'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='soft_nms',
n_calls=50,
)
for i, c in enumerate(production_countries_df.index.values):
production_countries_df.loc[c, "median_salary"] = train_df[train_df['isProductionCountry_' + c]==1].revenue.median()
production_countries_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 )<feature_engineering>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
best_sigma = opt_result.x[2]
print('-'*11 + 'SOFT NMS' + '-'*11)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Sigma]: {best_sigma:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )
for i, p in enumerate(production_countries_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopProductionCountry_' + p] = train_df['production_countries_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopProductionCountry_' + p] = test_df['production_countries_processed'].apply(lambda x: 1 if p in x else 0 )<feature_engineering>
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='non_maximum_weighted',
n_calls=50,
)
for i, p in enumerate(production_companies_df.index.values):
production_companies_df.loc[p, "median_salary"] = train_df[train_df['isProductionCompany_' + p]==1].revenue.median()
production_companies_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(5 )<feature_engineering>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
print('-'*13 + 'NMW' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )
for i, p in enumerate(production_companies_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopProductionCompany_' + p] = train_df['production_companies_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopProductionCompany_' + p] = test_df['production_companies_processed'].apply(lambda x: 1 if p in x else 0)
<define_variables>
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.25, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='weighted_boxes_fusion',
n_calls=50,
)
columns_for_training = ["log_budget", "log_popularity", "log_runtime", "day_of_week", "year", "month", "week_of_year", "season",
"num_genres", "num_of_production_countries", "log_num_of_cast", "log_num_of_male_cast", "log_num_of_female_cast", "has_collection",
"has_homepage", "has_tag", "is_english_language",
"log_num_of_crew", "log_num_of_male_crew", "log_num_of_female_crew",
"log_title_len", "log_overview_len", "log_tagline_len",
"log_num_of_directors", "log_num_of_producers", "log_num_of_editors", "log_num_of_art_crew", "log_num_of_sound_crew",
"log_num_of_costume_crew", "log_num_of_camera_crew", "log_num_of_visual_effects_crew", "log_num_of_lighting_crew",
"log_num_of_other_crew"]
columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopGenre_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopProductionCompany_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopProductionCountry_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_actor_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_keyword_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_director_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_producer_'), axis=1 ).columns.values)
 columns_for_training<count_missing_values>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
else:
best_final_score = 0.6953
best_iou_thr = 0.44
best_skip_box_thr = 0.43
print('-'*13 + 'WBF' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )
train_df[columns_for_training].isna().sum().sum()<prepare_x_and_y>
models = [
load_net('.. /input/effdet5-folds-v2/fold0-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold1-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold2-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold3-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold4-best.bin'),
load_net('.. /input/effdet5-folds-mixup/fold0-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold1-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold2-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold3-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold4-best-v2.bin'),
]
<split><EOS> 
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(
 images,
score_threshold=0.25, 
):
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )