Kaggle competition: New York City Taxi Fare Prediction
import numpy as np
import pandas as pd
import os<install_modules>
train = pd.read_csv('.. /input/train.csv', nrows=5000000)
 test = pd.read_csv('.. /input/test.csv')
 train.head()
!pip install -U /kaggle/input/orkatzfdata/torch-1.5.0+cu101-cp37-cp37m-linux_x86_64.whl /kaggle/input/orkatzfdata/torchvision-0.6.0+cu101-cp37-cp37m-linux_x86_64.whl<install_modules>
print(train.isnull().sum())
 train = train.dropna(how='any',axis=0 )
!pip install '/kaggle/input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl'
!pip install /kaggle/input/orkatzfdata/yacs-0.1.7-py3-none-any.whl
!mkdir fvcore
!cp -R '/kaggle/input/orkatzfdata/fvcore-0.1.dev200407/fvcore-0.1.dev200407/'./fvcore
!pip install fvcore/fvcore-0.1.dev200407/.
 !mkdir detectron2-ResNeSt
!cp -R /kaggle/input/orkatzfdata/detectron2-ResNeSt/*./detectron2-ResNeSt/
!pip install detectron2-ResNeSt/ .<set_options>
train['abs_diff_longitude'] = np.abs(train['dropoff_longitude'] - train['pickup_longitude'])
 train['abs_diff_latitude'] = np.abs(train['dropoff_latitude'] - train['pickup_latitude'])
 test['abs_diff_longitude'] = np.abs(test['dropoff_longitude'] - test['pickup_longitude'])
 test['abs_diff_latitude'] = np.abs(test['dropoff_latitude'] - test['pickup_latitude'] )
setup_logger() 
<load_pretrained>
train = train.loc[train['fare_amount']>0,:]
train = train.loc[(train["passenger_count"]<=6)&(train["passenger_count"]>0),:]
train = train.loc[(train["abs_diff_latitude"]<2)&(train["abs_diff_longitude"]<2),:]
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_cascade_rcnn_ResNeSt_101_FPN_syncbn_range-scale_1x.yaml"))
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.MODEL.WEIGHTS = os.path.join('/kaggle/input/model-pth2/', "model_fold2.pth")
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4
cfg.DATASETS.TEST =("m5_val",)
 predictor1 = DefaultPredictor(cfg )<load_from_csv>
train = train[train['fare_amount'].between(left = 2.5, right = 100)]
df_sub = pd.read_csv('/kaggle/input/global-wheat-detection/sample_submission.csv' )<define_variables>
train.loc[:,'timestamp_with_key'] = train.loc[:,'key']
test.loc[:,'timestamp_with_key'] = test.loc[:,'key']
train.key = pd.DataFrame({'key':train['key'].str.split('.' ).str[1].astype('int')})
 test.key = pd.DataFrame({'key':test['key'].str.split('.' ).str[1].astype('int')} )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<data_type_conversions>
results = []
for image_id in df_sub['image_id']:
im = cv2.imread('/kaggle/input/global-wheat-detection/test/{}.jpg'.format(image_id))
boxes = []
scores = []
labels = []
outputs = predictor1(im)
 out = outputs["instances"].to("cpu")
 scores = out.get_fields() ['scores'].numpy()
boxes = out.get_fields() ['pred_boxes'].tensor.numpy().astype(int)
 labels= out.get_fields() ['scores'].numpy()
boxes = boxes.astype(int)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result )<save_to_csv>
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'], infer_datetime_format=True ).dt.tz_localize('UTC')
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'], infer_datetime_format=True ).dt.tz_localize('UTC')
 train.loc[:,'hour_no'] = train['pickup_datetime'][:].dt.strftime('%-H' ).astype('int')
 test.loc[:,'hour_no'] = test['pickup_datetime'][:].dt.strftime('%-H' ).astype('int')
 train.loc[:,'weekday_no'] = train['pickup_datetime'][:].dt.strftime('%w' ).astype('int')
 test.loc[:,'weekday_no'] = test['pickup_datetime'][:].dt.strftime('%w' ).astype('int')
 train.loc[:,'day_no'] = train['pickup_datetime'][:].dt.strftime('%-d' ).astype('int')
 test.loc[:,'day_no'] = test['pickup_datetime'][:].dt.strftime('%-d' ).astype('int')
 train.loc[:,'year_no'] = train['pickup_datetime'][:].dt.strftime('%-y' ).astype('int')
 test.loc[:,'year_no'] = test['pickup_datetime'][:].dt.strftime('%-y' ).astype('int' )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<install_modules>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
def dist_haversine(x):
R = 6371
picklat = math.radians(x[1])
 droplat = math.radians(x[3])
 latdiff = abs(droplat-picklat)
 picklon = math.radians(x[0])
 droplon = math.radians(x[2])
 londiff = abs(droplon-picklon)
 a = math.sin(latdiff/2)* math.sin(latdiff/2)+\
math.cos(picklat)* math.cos(droplat)*\
math.sin(londiff/2)* math.sin(londiff/2)
 c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
return(R * c)
 train['dist_haversine_km'] = pd.DataFrame(list(
 map(lambda x: dist_haversine(x), train[["pickup_longitude","pickup_latitude","dropoff_longitude","dropoff_latitude"]].values)) ,
index=train.index)
 test['dist_haversine_km'] = pd.DataFrame(list(
 map(lambda x: dist_haversine(x), test[["pickup_longitude","pickup_latitude","dropoff_longitude","dropoff_latitude"]].values)) ,
index=test.index)
USE_OPTIMIZE = len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) == 10
USE_OPTIMIZE<load_from_csv>
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
train['fare_per_km'] = train['fare_amount']/(train["dist_haversine_km"])
 train['fare_per_km_passenger'] = train['fare_amount']/(train['dist_haversine_km']*train['passenger_count'] )
%%time
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']]
df_folds['bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds['source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds['stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))
df_holdout = df_folds.iloc[holdout_index].copy()
df_folds = df_folds.iloc[train_index].copy()
df_folds['fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<load_pretrained>
train.groupby('key' ).agg({'fare_per_km_passenger':'mean','key':'count','passenger_count':'mean','fare_amount':'mean'} )
holdout_dataset = DatasetRetriever(
 image_ids=df_holdout.index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 holdout_loader = DataLoader(
 holdout_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<load_pretrained>
train.loc[train['fare_per_km_passenger']>20,['fare_per_km_passenger','fare_amount','dist_haversine_km']]
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
if USE_OPTIMIZE:
models = [
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold0-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold1-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold2-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold3-best.bin'),
load_net('.. /input/wheat-skopt-holdout-effdet5-folds/fold4-best.bin'),
]<categorify>
del reindexed, grouped, get_max_count, grouped_df
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
if USE_OPTIMIZE:
all_predictions = []
for images, targets, image_ids in tqdm(holdout_loader, total=len(holdout_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
fold_predictions = {}
for fold_number in range(5):
fold_predictions[fold_number] = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
image_predictions = {
'image_id': image_ids[i],
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
}
for fold_number in range(5):
boxes, scores = process_det(i, fold_predictions[fold_number])
 image_predictions[f'pred_boxes_fold{fold_number}'] = boxes
image_predictions[f'scores_fold{fold_number}'] = scores
all_predictions.append(image_predictions )<normalization>
train = train.loc[ ~(((train['dist_haversine_km']>1)& train['fare_per_km']<0.2)) ]
train = train.loc[~(( train['dist_haversine_km']<0.01)&(train['fare_per_km']>50)) ]
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )<define_variables>
train = train.drop(['fare_per_km_passenger','fare_per_km'],axis=1 )
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr,
method,
sigma=0.5, 
):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for fold_number in range(5):
pred_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy()
scores = all_predictions[i][f'scores_fold{fold_number}'].copy()
folds_boxes.append(pred_boxes)
 folds_scores.append(scores)
 folds_labels.append(np.ones(pred_boxes.shape[0]))
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<compute_train_metric>
train = train.loc[train['pickup_latitude'].between(40, 42)]
train = train.loc[train['pickup_longitude'].between(-75, -72)]
train = train.loc[train['dropoff_latitude'].between(40, 42)]
train = train.loc[train['dropoff_longitude'].between(-75, -72)]
%%time
if USE_OPTIMIZE:
print('[WBF]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='weighted_boxes_fusion', 
))
print('[NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='nms', 
))
print('[SOFT NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
sigma=0.1,
method='soft_nms', 
))
print('[NMW]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='non_maximum_weighted', 
))<find_best_params>
train.loc[:,'pickuplat_no'], pick_lat_bin = pd.cut(train['pickup_latitude'],100, labels=False, retbins=True)
 train.loc[:,'pickuplong_no'], pick_long_bin = pd.cut(train['pickup_longitude'],100, labels=False, retbins=True)
 train.loc[:,'dropofflat_no'], drop_lat_bin = pd.cut(train['dropoff_latitude'],100, labels=False, retbins=True)
 train.loc[:,'dropofflong_no'], drop_long_bin = pd.cut(train['dropoff_longitude'],100, labels=False, retbins=True)
 test['pickuplat_no'] = pd.cut(test['pickup_latitude'], pick_lat_bin, labels=False ).fillna(int(train.loc[:,'pickuplat_no'].mean()))
test['pickuplong_no'] = pd.cut(test['pickup_longitude'], pick_long_bin, labels=False ).fillna(int(train.loc[:,'pickuplong_no'].mean()))
test['dropofflat_no'] = pd.cut(test['dropoff_latitude'], drop_lat_bin, labels=False ).fillna(int(train.loc[:,'dropofflat_no'].mean()))
test['dropofflong_no'] = pd.cut(test['dropoff_longitude'], drop_long_bin, labels=False ).fillna(int(train.loc[:,'dropofflong_no'].mean()))
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + f'{method}' + '-'*5)
 log(params)
 final_score = calculate_final_score(all_predictions, method=method, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<normalization>
train.loc[:,'pickdrop_lat_diff'] = abs(train['pickuplat_no'].astype(int)- train['dropofflat_no'].astype(int))
train.loc[:,'pickdrop_long_diff'] = abs(train['pickuplong_no'].astype(int)- train['dropofflong_no'].astype(int))
train.loc[:,'final_dist_factor'] =(train['pickdrop_lat_diff'].astype(int)+ train['pickdrop_long_diff'].astype(int))
test.loc[:,'pickdrop_lat_diff'] = abs(test['pickuplat_no'].astype(int)- test['dropofflat_no'].astype(int))
test.loc[:,'pickdrop_long_diff'] = abs(test['pickuplong_no'].astype(int)- test['dropofflong_no'].astype(int))
test.loc[:,'final_dist_factor'] =(test['pickdrop_lat_diff'].astype(int)+ test['pickdrop_long_diff'].astype(int))
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='nms',
n_calls=50,
)<find_best_params>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
print('-'*13 + 'NMS' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )<find_best_params>
def calc_cwd_factor(df, col):
new_df = df.groupby(col)['key'].count().sort_values(ascending=False ).reset_index()
new_df['cwd_factor'] = 1
count = 1
for i in range(1,new_df.shape[0]):
count += 1
if new_df.loc[i-1,'key'] == new_df.loc[i,'key']:
count -= 1
new_df.loc[i,'cwd_factor'] = count
new_df.index = new_df[col]
return new_df
fact_df = calc_cwd_factor(train, 'pickuplat_no')
 train.loc[:,'pickuplat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['pickuplat_no']))
fact_df = calc_cwd_factor(train, 'pickuplong_no')
 train.loc[:,'pickuplong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['pickuplong_no']))
fact_df = calc_cwd_factor(train, 'dropofflat_no')
 train.loc[:,'dropofflat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['dropofflat_no']))
fact_df = calc_cwd_factor(train, 'dropofflong_no')
 train.loc[:,'dropofflong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], train['dropofflong_no']))
fact_df = calc_cwd_factor(test, 'pickuplat_no')
 test.loc[:,'pickuplat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['pickuplat_no']))
fact_df = calc_cwd_factor(test, 'pickuplong_no')
 test.loc[:,'pickuplong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['pickuplong_no']))
fact_df = calc_cwd_factor(test, 'dropofflat_no')
 test.loc[:,'dropofflat_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['dropofflat_no']))
fact_df = calc_cwd_factor(test, 'dropofflong_no')
 test.loc[:,'dropofflong_cwd_factor'] = list(map(lambda x: fact_df.loc[x,'cwd_factor'], test['dropofflong_no']))
del fact_df
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
Real(0.0, 1, name='sigma'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='soft_nms',
n_calls=50,
)<find_best_params>
gc.collect()
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
best_sigma = opt_result.x[2]
print('-'*11 + 'SOFT NMS' + '-'*11)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Sigma]: {best_sigma:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )<define_search_space>
import sklearn
from sklearn import *
from sklearn.preprocessing import Normalizer
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.0, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='non_maximum_weighted',
n_calls=50,
)<find_best_params>
train = shuffle(train.iloc[:,:] ).reset_index(drop=True)
 val = train.iloc[int(0.9*train.shape[0]):,:]
train = train.iloc[:int(0.9*train.shape[0]),:]
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
print('-'*13 + 'NMW' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )<find_best_params>
object_cols = ['key','pickup_datetime','timestamp_with_key']
categorical_cols = ['passenger_count', 'hour_no', 'weekday_no', 'day_no', 'year_no', 'pickuplat_no', 'pickuplong_no',
'dropofflat_no', 'dropofflong_no', 'pickdrop_lat_diff', 'pickdrop_long_diff', 'final_dist_factor',
'pickuplat_cwd_factor', 'pickuplong_cwd_factor', 'dropofflat_cwd_factor', 'dropofflong_cwd_factor']
numerical_cols =['pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude',
'abs_diff_longitude', 'abs_diff_latitude', 'dist_haversine_km']
target = ['fare_amount']
cor = train.loc[:,numerical_cols+target]
f, ax = plt.subplots(1,1,figsize=(18,7))
sns.heatmap(cor.corr() , annot=True )
%%time
space = [
Real(0, 1, name='iou_thr'),
Real(0.25, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='weighted_boxes_fusion',
n_calls=50,
)<find_best_params>
gc.collect()
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
else:
best_final_score = 0.6953
best_iou_thr = 0.44
best_skip_box_thr = 0.43
print('-'*13 + 'WBF' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )<load_pretrained>
f_reg = f_regression(train[categorical_cols+numerical_cols],train['fare_amount'], center=True)
 mir_c = mutual_info_regression(train[categorical_cols][:10000], train['fare_amount'][:10000])
 mir_n = mutual_info_regression(train[numerical_cols][:10000], train['fare_amount'][:10000])
 impt_cols_freg = list(zip(f_reg[0], categorical_cols+numerical_cols))
impt_cols_freg = sorted(impt_cols_freg, key = lambda x:x[0], reverse=True)
 impt_cols_c = list(zip(mir_c ,categorical_cols))
impt_cols_c = sorted(impt_cols_c, key = lambda x:x[0], reverse=True)
 impt_cols_n = list(zip(mir_n ,numerical_cols))
impt_cols_n = sorted(impt_cols_n, key = lambda x:x[0], reverse=True )
models = [
load_net('.. /input/effdet5-folds-v2/fold0-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold1-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold2-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold3-best.bin'),
load_net('.. /input/effdet5-folds-v2/fold4-best.bin'),
load_net('.. /input/effdet5-folds-mixup/fold0-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold1-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold2-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold3-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold4-best-v2.bin'),
]<data_type_conversions>
drop_cols = []
for i in impt_cols_c[-8:]:
drop_cols.append(i[1])
 train = train.drop(drop_cols, axis=1)
 test = test.drop(drop_cols, axis=1 )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(
 images,
score_threshold=0.25, 
):
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<install_modules>
gc.collect()
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<load_from_csv>
train_y = train['fare_amount'][:]
val_y = val['fare_amount'][:]
cols = [i for i in train.columns if i not in ['fare_amount'] + object_cols]
train_x = train.loc[:, cols]
val_x = val.loc[:, cols]
test_x = test.loc[:,cols]
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
del train, val
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<categorify>
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
def collate_fn(batch):
return tuple(zip(*batch))
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes<create_dataframe>
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'nthread': 4,
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
'num_rounds':50000
}
cols = list(train_x.select_dtypes(exclude=['float64'] ).columns)
 train_set = lgbm.Dataset(train_x, train_y, silent=False,categorical_feature=cols)
 valid_set = lgbm.Dataset(val_x, val_y, silent=False,categorical_feature=cols)
 model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=500,verbose_eval=500, valid_sets=valid_set )
dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 0].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True,
)<load_pretrained>
pred_train = model.predict(train_x, num_iteration = model.best_iteration)
 pred_val = model.predict(val_x, num_iteration = model.best_iteration)
 pred_test = model.predict(test_x, num_iteration = model.best_iteration )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/effdet5-folds-mixup/fold0-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold1-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold2-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold3-best-v2.bin'),
load_net('.. /input/effdet5-folds-mixup/fold4-best-v2.bin'),
]<load_pretrained>
gc.collect()
%%time
all_predictions = []
for fold_number in range(5):
validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 validation_loader = DataLoader(
 validation_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn 
)
 for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
det = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
all_predictions.append({
'pred_boxes':(boxes*2 ).clip(min=0, max=1023 ).astype(int),
'scores': scores,
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
'image_id': image_ids[i],
} )<normalization>
rmse_train = np.sqrt(mean_squared_error(train_y, pred_train))
rmse_val = np.sqrt(mean_squared_error(val_y, pred_val))
print(rmse_train, rmse_val )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )<compute_train_metric>
final = pd.DataFrame({'key':test.timestamp_with_key, 'fare_amount':pred_test}, columns = ['key', 'fare_amount'])
 final.to_csv('submission_lgb.csv', index = False)
 final.head()