Kaggle competition: New York City Taxi Fare Prediction
%%time
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = False
models = []
for w in weights:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
for model in models:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, 0.4, opt.iou_thres,fast=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
class opt:
weights_folds = list(glob.glob(".. /input/yolov5pth/*"))
weights_folds.sort()
weights = weights_folds[4:]
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()<compute_train_metric>
gc.collect()
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.34, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels<define_variables>
imp_locations = [(40.639722, -73.778889),(40.6925, -74.168611), 
(40.77725, -73.872611)]
all_path,all_score,all_bboxex = res<categorify>
cal = USFederalHolidayCalendar()
holidays = pd.to_datetime(cal.holidays(start='2009-01-01', end='2015-12-31'))
def haverSine(df):
Radius = 6371
h = np.sqrt(( np.sin(df['abs_latitude_diff']/2)*
np.sin(df['abs_latitude_diff']/2)) + 
(np.cos(df['pickup_latitude'])*
np.cos(df['dropoff_latitude'])
 *(np.sin(df['abs_longitude_diff']/2)*
np.sin(df['abs_longitude_diff']/2))))
df['haversine'] = 2*Radius*np.arcsin(h)
 def calcHaversine(lat1, long1, lat2, long2):
Radius = 6371
h = np.sqrt(( np.sin(( lat1 - lat2)/2)*
np.sin(( lat1- lat2)/2)) + 
(np.cos(lat1)*
np.cos(lat2)
 *(np.sin(( long1 - long2)/2)*
np.sin(( long1 - long2)/2))))
return 2*np.arcsin(h)
 def getDateTime(df):
df['datetime'] = pd.to_datetime(df['pickup_datetime'],
format = "%Y-%m-%d %H:%M:%S %Z")
 def getYear(df):
df['year'] = df['datetime'].dt.year
def getTime(df):
df['time'] = df['datetime'].dt.hour
def getMonth(df):
df['month'] = df['datetime'].dt.month
def getWeekday(df):
df['weekday'] = df['datetime'].dt.weekday
def getDayOrNight(df):
df['light'] = df['time'].apply(lambda x:"D" if(x >5 and x <18)
 else "A" if(x >=18 and x<=23)else "N")
 def getLightEncoding(df):
df['light_d'] = df['light'].apply(lambda x:1 if x == "D" else 0)
 df['light_a'] = df['light'].apply(lambda x:1 if x == "N" else 0)
 df['light_n'] = df['light'].apply(lambda x:1 if x == "a" else 0)
 def distFromImpLocations(df):
for i in range(len(imp_locations)) :
df['imp_'+str(i)+"a"] = calcHaversine(df.dropoff_latitude,
df.dropoff_longitude,
imp_locations[i][0], imp_locations[i][1])
 df['imp_'+str(i)+"b"] = calcHaversine(df.dropoff_latitude,
df.dropoff_longitude,
imp_locations[i][0], imp_locations[i][1])
 def isHoliday(df):
df['isHoliday'] = df['datetime'].apply(lambda x:1 if x.date() in holidays
else 0)
results =[]
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 !rm -rf *
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
train['abs_longitude_diff'] =(train['pickup_longitude'] -
train['dropoff_longitude'] ).abs()
train['abs_latitude_diff'] =(train['pickup_latitude'] -
train['dropoff_latitude'] ).abs()
haverSine(train)
 getDateTime(train)
 getYear(train)
 getMonth(train)
 getTime(train)
 getWeekday(train)
 getDayOrNight(train)
 distFromImpLocations(train)
 isHoliday(train )
import numpy as np
import pandas as pd
import os 
<init_hyperparams>
train.isnull().sum()
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = False
model = torch.load(weights, map_location=device)['model'].to(device ).float().eval()
dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
print(im0s.shape)
 img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
if True:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, 0.4, opt.iou_thres,fast=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
if __name__ == '__main__':
class opt:
weights = ".. /input/yolov5pth/best_yolov5x_fold1.pt"
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 print(opt)
 with torch.no_grad() :
res = detect()<compute_train_metric>
train.loc[train['passenger_count'] ==0 ,'passenger_count'
]= train['passenger_count'].mode() [0]
train.loc[train['haversine'] == 0, 'haversine'
] = train['haversine'].mean()
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.34, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels<define_variables>
print("OldSize = ", train.shape)
 train1 = train.loc[(train.abs_longitude_diff < 5.0)& 
(train['abs_latitude_diff'] < 5.0)& 
(train['fare_amount'] > 0.0)& 
(train.fare_amount<100.0)& 
(train.pickup_longitude<=-71.0)& 
(train.pickup_longitude>=-75)& 
(train.pickup_latitude>=40.0)& 
(train.pickup_latitude<=42.0)& 
(train.dropoff_longitude>=-75.0)& 
(train.dropoff_longitude<=-71.0)& 
(train.dropoff_latitude<=42.0)& 
(train.dropoff_latitude>=40.0),:]
print("NewSize = ",train1.shape )
all_path,all_score,all_bboxex = res<categorify>
getLightEncoding(train1 )
results =[]
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'] )<save_to_csv>
train2 = np.column_stack(( train1['abs_longitude_diff'],
train1['abs_longitude_diff']**2,
train1['abs_latitude_diff'],
train1['abs_latitude_diff']**2,
train1.haversine,
train1['haversine']**2,
train1.pickup_longitude,
train1.pickup_latitude,
train1.dropoff_longitude,
train1.dropoff_latitude,
train1.imp_0a,
train1.imp_1a,
train1.imp_2a,
train1.light_d,
train1.light_a,
train1.weekday,
train1.year,
train1.month,
train1.passenger_count))
FA = train1['fare_amount'].values
print(train2.shape)
 print(FA.shape )
test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
def normalize(sigma, mean, X):
X =(X - mean)/sigma
return X
train2[:,0:6] = normalize(sigma, mean, train2[:,0:6] )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
train_X, val_X, train_Y, val_Y = train_test_split(train2, FA, test_size = 0.001,
random_state = 3)
 print(train_X.shape, val_X.shape, train_Y.shape, val_Y.shape )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
def rmse(pred, actual):
assert(len(pred)== len(actual))
return np.sqrt(((pred - actual)**2 ).sum() /len(pred))
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
lgbmObj = lgbm.LGBMRegressor(n_estimators=2000,
learning_rate = 0.03,
subsample=0.98,
subsample_freq=3,
reg_lambda = 0.0,
max_depth=7,
random_state=42)
 lgbmObj.fit(train_X, train_Y,eval_set=[(train_X,train_Y),(val_X,val_Y)],
eval_metric='rmse',verbose=False )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
print("Training Error = ", rmse(lgbmObj.predict(train_X), train_Y))
print("Validation Error = ", rmse(lgbmObj.predict(val_X), val_Y))
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin' )<categorify>
test = pd.read_csv(".. /input/test.csv")
 print(test.shape )
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
test.isnull().sum()
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<concatenate>
test['abs_longitude_diff'] =(test['pickup_longitude']
- test['dropoff_longitude'] ).abs()
test['abs_latitude_diff'] =(test['pickup_latitude']
- test['dropoff_latitude'] ).abs()
haverSine(test)
 getDateTime(test)
 getYear(test)
 getMonth(test)
 getTime(test)
 getWeekday(test)
 getDayOrNight(test)
 distFromImpLocations(test)
 isHoliday(test )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
test.loc[test['passenger_count'] ==0 ,'passenger_count'
]= test['passenger_count'].mode() [0]
test.loc[test['haversine'] == 0, 'haversine'
] = test['haversine'].mean()
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
getLightEncoding(test)
 test.head()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
test_X = np.column_stack(( test.abs_longitude_diff,
test['abs_longitude_diff']**2,
test.abs_latitude_diff,
test['abs_latitude_diff']**2,
test['haversine'],
test['haversine']**2,
test.pickup_longitude,
test.pickup_latitude,
test.dropoff_longitude,
test.dropoff_latitude,
test.imp_0a,
test.imp_1a,
test.imp_2a,
test.light_d,
test.light_a,
test.weekday,
test.year,
test.month,
test.passenger_count))
print(test_X.shape )
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
test_X[:,:6] = normalize(sigma, mean, test_X[:,:6] )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
prediction = lgbmObj.predict(test_X)
 sub = pd.DataFrame()
sub['key'] = test.key
sub['fare_amount'] = prediction
sub.to_csv("prediction.csv", index = False )