Kaggle competition: New York City Taxi Fare Prediction
df_submission = pd.read_csv(".. /input/abstraction-and-reasoning-challenge/sample_submission.csv")
 df_submission.head()<string_transform>
train_df_v1 = pd.read_csv('.. /input/train.csv', nrows = 10000000 , parse_dates= ['key'] )
for i, row in df_submission.iterrows() :
cand_0 = ''
cand_1 = ''
cand_2 = '|' + str(rd_test[i] ).replace('[','' ).replace(']','' ).replace('
','|' ).replace(' ','')+ '|'
answer = ''
for j, cand in enumerate(row[1].split(' ')) :
if j == 0:
cand_0 = cand
answer += cand_0
elif j == 1:
nums = []
for c in cand_0.replace('|', ''):
nums.append(int(c))
for k, c in enumerate(cand_0):
if c == '|':
cand_1 += '|'
elif int(c)== np.amax(nums):
if np.amax(rd_test[i])== cand_2[k]:
cand_1 += c
else:
cand_1 += cand_2[k]
else:
cand_1 += c
answer += ' ' + cand_1
elif j == 2:
answer += ' ' + cand_2
df_submission.at[i,'output'] = answer<save_to_csv>
def add_travel_vector_features(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
add_travel_vector_features(train_df_v1 )
df_submission.to_csv("submission.csv", index=False )<import_modules>
def add_date_data(df):
train_df_v1['weekday']=train_df_v1.key.apply(lambda x:calendar.day_name[x.weekday() ])
 train_df_v1['hour'] = train_df_v1.key.map(lambda x: x.hour)
 train_df_v1['day'] = train_df_v1.key.map(lambda x: x.day)
 train_df_v1['month'] = train_df_v1.key.map(lambda x: x.month)
 train_df_v1['year'] = train_df_v1.key.map(lambda x: x.year)
 add_date_data(train_df_v1 )
import numpy as np
import pandas as pd
import os
from tqdm.auto import tqdm
import shutil as sh<install_modules>
print('Old size: %d' % len(train_df_v1))
train_df_v1 = train_df_v1.dropna(how = 'any', axis = 'rows')
 features_v1_list = ['pickup_longitude','pickup_latitude', 'dropoff_longitude',
'dropoff_latitude','passenger_count', 'abs_diff_longitude', 'abs_diff_latitude'
,'hour','day','month','year','weekday']
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_latitude'] < 1 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_longitude'] < 1 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_latitude'] > 0.001 ]
train_df_v1 = train_df_v1.loc[train_df_v1['abs_diff_longitude'] > 0.001 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_longitude'] < -72 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_longitude'] > -75 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_latitude'] < 41 ]
train_df_v1 = train_df_v1.loc[train_df_v1['pickup_latitude'] > 39 ]
train_df_v1 = train_df_v1.loc[train_df_v1['fare_amount'] > 1 ]
print('New size: %d' % len(train_df_v1))
train_df_v1_features = train_df_v1[features_v1_list]
!cp -r.. /input/yolov5train/* .<install_modules>
class MultiColumnLabelEncoder:
def __init__(self,columns = None):
self.columns = columns
def fit(self,X,y=None):
return self
def transform(self,X):

 output = X.copy()
if self.columns is not None:
for col in self.columns:
output[col] = LabelEncoder().fit_transform(output[col])
 else:
for colname,col in output.iteritems() :
output[colname] = LabelEncoder().fit_transform(col)
 return output
def fit_transform(self,X,y=None):
return self.fit(X,y ).transform(X )
!pip install --no-deps '.. /input/weightedboxesfusion/' > /dev/null<feature_engineering>
train_df_v1_features_new = MultiColumnLabelEncoder(columns = ['weekday'] ).fit_transform(train_df_v1_features)
 def dist(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
distance = np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 return distance
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name,mini in tqdm(df.groupby('image_id')) :
if name in val_index:
path2save = 'val2017/'
else:
path2save = 'train2017/'
if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)) :
os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))
sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<compute_train_metric>
label_v1 = train_df_v1['fare_amount']
def run_wbf(boxes, scores, image_size=1023, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(im0, imgsz, model, device, conf_thres, iou_thres):
img = letterbox(im0, new_shape=imgsz)[0]
img = img[:, :, ::-1].transpose(2, 0, 1)
 img = np.ascontiguousarray(img)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=False)[0]
pred = non_max_suppression(pred, conf_thres, iou_thres)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<load_pretrained>
def XGB_regressor(train_X, train_y, feature_names=None, seed_val=2017, num_rounds=500):
param = {}
param['objective'] = 'reg:linear'
param['eta'] = 0.05
param['max_depth'] = 8
param['silent'] = 1
param['eval_metric'] = 'rmse'
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 model = xgb.train(plst, xgtrain, num_rounds)
 return model
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = '.. /input/yolosub/gwd/yolo5x_panet_1.pt'
imgsz = 1024
conf_thres = 0.5
iou_thres = 0.6
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for name in imagenames:
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
if im01.shape[0]!=1024 or im01.shape[1]!=1024:
continue
assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.6, skip_box_thr=0.43)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<load_pretrained>
model = XGB_regressor(train_X = train_df_v1_features_new , train_y = label_v1 )
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
pass
else:
!python train.py --img 1024 --batch 4 --epochs 10 --data.. /input/configyolo5/wheat0.yaml --cfg.. /input/yolosub/gwd/yolo5_PANET.yaml --weights.. /input/yolosub/gwd/yolo5x_panet_1.pt
!rm -rf convertor<define_variables>
xgtrain = xgb.DMatrix(train_df_v1_features_new, label=label_v1 )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<load_pretrained>
preds_v1 = model.predict(xgtrain )
def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = '.. /input/yolosub/gwd/yolo5x_panet_1.pt'
if not os.path.exists(weights):
weights = '.. /input/yolosub/gwd/yolo5x_panet_1.pt'
imgsz = 1024
conf_thres = 0.5
iou_thres = 0.6
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for name in imagenames:
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 if 1:
enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.6, skip_box_thr=0.5)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
if count<10:
for box, score in zip(boxes,scores):
cv2.rectangle(im0, 
(box[0], box[1]), 
(box[2]+box[0], box[3]+box[1]), 
(220, 0, 0), 2)
 cv2.putText(im0, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,
0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(im0)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
rmse = np.sqrt(sckit.metrics.mean_squared_error(label_v1, preds_v1))
print("RMSE: %f" %(rmse))
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
test_df_v1 = pd.read_csv('.. /input/test.csv' )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
testKey = pd.read_csv('.. /input/test.csv' )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
test_df_v1_fordates = test_df_v1
test_df_v1_fordates['key']= pd.to_datetime(test_df_v1_fordates['key'] )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
test_df_v1_fordates['hour'] = test_df_v1_fordates.key.map(lambda x: x.hour)
 test_df_v1_fordates['day'] = test_df_v1_fordates.key.map(lambda x: x.day)
 test_df_v1_fordates['month'] = test_df_v1_fordates.key.map(lambda x: x.month)
 test_df_v1_fordates['year'] = test_df_v1_fordates.key.map(lambda x: x.year)
 test_df_v1_fordates['weekday']=test_df_v1_fordates.key.apply(lambda x:calendar.day_name[x.weekday() ])
 add_travel_vector_features(test_df_v1_fordates)
 test_df_v1_features = test_df_v1_fordates[features_v1_list]
test_df_v1_features = MultiColumnLabelEncoder(columns = ['weekday'] ).fit_transform(test_df_v1_features )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
test_df_v1_features['distance_to_center'] = dist(nyc[1], nyc[0],
test_df_v1_features['pickup_latitude'], test_df_v1_features['pickup_longitude'])
 test_df_v1_features['pickup_distance_to_jfk'] = dist(jfk[1], jfk[0],
test_df_v1_features['pickup_latitude'], test_df_v1_features['pickup_longitude'])
 test_df_v1_features['dropoff_distance_to_jfk'] = dist(jfk[1], jfk[0],
test_df_v1_features['dropoff_latitude'], test_df_v1_features['dropoff_longitude'])
 test_df_v1_features['pickup_distance_to_ewr'] = dist(ewr[1], ewr[0],
test_df_v1_features['pickup_latitude'], test_df_v1_features['pickup_longitude'])
 test_df_v1_features['dropoff_distance_to_ewr'] = dist(ewr[1], ewr[0],
test_df_v1_features['dropoff_latitude'], test_df_v1_features['dropoff_longitude'])
 test_df_v1_features['pickup_distance_to_lgr'] = dist(lgr[1], lgr[0],
test_df_v1_features['pickup_latitude'], test_df_v1_features['pickup_longitude'])
 test_df_v1_features['dropoff_distance_to_lgr'] = dist(lgr[1], lgr[0],
test_df_v1_features['dropoff_latitude'], test_df_v1_features['dropoff_longitude'] )
class CrossEntropyLabelSmooth(nn.Module):

 def __init__(self, num_classes, epsilon=0.1, use_gpu=True):
super(CrossEntropyLabelSmooth, self ).__init__()
self.num_classes = num_classes
self.epsilon = epsilon
self.use_gpu = use_gpu
self.logsoftmax = nn.LogSoftmax(dim=1)
 def forward(self, inputs, targets):

 log_probs = self.logsoftmax(inputs)
 targets = torch.zeros(log_probs.size() ).scatter_(1, targets.unsqueeze(1 ).data.cpu() , 1)
 if self.use_gpu: targets = targets.cuda()
targets =(1 - self.epsilon)* targets + self.epsilon / self.num_classes
loss =(- targets * log_probs ).mean(0 ).sum()
return loss<compute_train_metric>
xgtrainTest = xgb.DMatrix(test_df_v1_features )
def fastrcnn_loss(class_logits, box_regression, labels, regression_targets):

 labels = torch.cat(labels, dim=0)
 regression_targets = torch.cat(regression_targets, dim=0)
 labal_smooth_loss = CrossEntropyLabelSmooth(2)
 classification_loss = labal_smooth_loss(class_logits, labels)
 sampled_pos_inds_subset = torch.nonzero(labels > 0 ).squeeze(1)
 labels_pos = labels[sampled_pos_inds_subset]
N, num_classes = class_logits.shape
box_regression = box_regression.reshape(N, -1, 4)
 box_loss = det_utils.smooth_l1_loss(
 box_regression[sampled_pos_inds_subset, labels_pos],
regression_targets[sampled_pos_inds_subset],
beta=1 / 9,
size_average=False, 
)
 box_loss = box_loss / labels.numel()
return classification_loss, box_loss<choose_model_class>
preds_v1_test = model.predict(xgtrainTest )
def resnest_fpn_backbone(pretrained, norm_layer=misc_nn_ops.FrozenBatchNorm2d, trainable_layers=3):
backbone = resnest101e(pretrained=pretrained)
 assert trainable_layers <= 5 and trainable_layers >= 0
layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]
for name, parameter in backbone.named_parameters() :
if all([not name.startswith(layer)for layer in layers_to_train]):
parameter.requires_grad_(False)
 return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
in_channels_stage2 = backbone.inplanes // 8
in_channels_list = [
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
]
out_channels = 256
return BackboneWithFPN(backbone, return_layers, in_channels_list, out_channels )<choose_model_class>
submission = pd.DataFrame(
 {'key': testKey.key, 'fare_amount': preds_v1_test },
columns = ['key', 'fare_amount'])
 submission.to_csv('submissionXGlast.csv', index = False )