Kaggle competition: Bike Sharing Demand
<SOS> metric: RMSLE Kaggle data source: bike-sharing-demand<define_variables>
import os
import datetime
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
train = pd.read_csv('/kaggle/input/bike-sharing-demand/train.csv')
 test = pd.read_csv('/kaggle/input/bike-sharing-demand/test.csv')
 train.head()
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
image_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p )<prepare_output>
train['datetime'] = pd.to_datetime(train['datetime'])
 test['datetime'] = pd.to_datetime(test['datetime'])
 train.head()
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df.head()<feature_engineering>
def split_datetime(df):
df['year'] = df['datetime'].apply(lambda t: t.year)
 df['month'] = df['datetime'].apply(lambda t: t.month)
 df['day'] = df['datetime'].apply(lambda t: t.day)
 df['dayofweek'] = df['datetime'].apply(lambda t: t.dayofweek)
 df['hour'] = df['datetime'].apply(lambda t: t.hour)
 df = df.drop(['datetime'], axis=1)
 return df
results_df['image_id'] = results_df['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg')
 results_df['image_id']<feature_engineering>
test = split_datetime(test)
 train = split_datetime(train)
 train = train.drop(['casual', 'registered'], axis=1)
 train.head()
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_ROOT_PATH+'/'+ x+'.jpg' )<concatenate>
train['count'] = np.log1p(train['count'] )
train_data_plabel = pd.concat([results_df,marking], axis=0)
 train_data_plabel<load_pretrained>
train = train[np.abs(train['count']-train['count'].mean())<=(3*train['count'].std())]
test_ids_ = os.listdir(".. /input/global-wheat-detection/test")
 test_idss = []
for test_id in test_ids_:
ids = test_id[:-4]
test_idss.append(ids)
 print(test_idss)
 test_images = {}
test_ids = []
for t_id in test_idss:
test_image = cv2.imread(f'.. /input/global-wheat-detection/test/{t_id}.jpg', cv2.IMREAD_COLOR)
 test_images[t_id] = test_image
test_ids.append(t_id)
 print(len(test_images))
test_ids = pd.DataFrame(test_ids,columns=['image_id'])
 print(test_ids)
 test_ids['image_id'] = test_ids['image_id'].apply(lambda x: DATA_ROOT_PATH+'/'+ x+'.jpg' )<feature_engineering>
drop_idx = train[(train['atemp'] > 20)&(train['atemp'] < 40)&(train['temp'] > 10)&(train['temp'] < 20)].index
train = train.drop(drop_idx )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<define_variables>
def scaling(df):
scaler = MinMaxScaler()
num_cols = ['temp', 'atemp', 'humidity', 'windspeed']
df[num_cols] = scaler.fit_transform(df[num_cols])
 return df
fold_number = 0
train_ids = df_folds[df_folds['fold'] != fold_number].index.values
valid_ids = df_folds[df_folds['fold'] == fold_number].index.values 
<import_modules>
train = scaling(train)
 test = scaling(test )
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes<create_dataframe>
X_train, X_test, y_train, y_test = train_test_split(train.drop(['count'], axis=1), train['count'], test_size=0.3 )
train_dataset = DatasetRetriever(
 image_ids=train_ids,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=valid_ids,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True,
)<load_pretrained>

 def rmsle(y, pred):
log_y = np.log1p(y)
 log_pred = np.log1p(pred)
 squared_error =(log_y - log_pred)** 2
rmsle = np.sqrt(np.mean(squared_error))
return rmsle
def collate_fn(batch):
return tuple(zip(*batch))
train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<choose_model_class>
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.model_selection import GridSearchCV
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(effdet_weights)
 net.load_state_dict(checkpoint['model_state_dict'])
 return DetBenchTrain(net, config)
 net = get_net()<normalization>
def evaluate(reg_cls, params=None):
reg = reg_cls()
if params:
reg = GridSearchCV(reg, param_grid=params, refit=True)
 reg.fit(X_train, y_train)
 pred = reg.predict(X_test)
 y_test_exp = np.expm1(y_test)
 pred_exp = np.expm1(pred)
 print('
', reg_cls)
 if params:
print(reg.best_params_)
 reg = reg.best_estimator_
print(rmsle(y_test_exp, pred_exp))
return reg, pred_exp
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<define_variables>
lr_reg, pred_lr = evaluate(LinearRegression)
 rg_reg, pred_rg = evaluate(Ridge)
 ls_reg, pred_ls = evaluate(Lasso)
 rf_reg, pred_rf = evaluate(RandomForestRegressor)
 gb_reg, pred_gb = evaluate(GradientBoostingRegressor)
 xg_reg, pred_xg = evaluate(XGBRegressor)
 lg_reg, pred_lg = evaluate(LGBMRegressor)
 params = {'n_estimators': [100*i for i in range(1, 6)]}
xg_reg, pred_xg = evaluate(XGBRegressor, params)
 lg_reg, pred_lg = evaluate(LGBMRegressor, params )
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >1<train_model>
feature_importances(xg_reg )
if is_TEST:
run_training()
else:
pass<load_pretrained>
submission = pd.read_csv('.. /input/bike-sharing-demand/sampleSubmission.csv')
 submission
weights = f'plabel_model/last-checkpoint1.bin'
if not os.path.exists(weights):
weights = effdet_weights
net = load_net(weights )<normalization>
pred = lg_reg.predict(test)
 pred_exp = np.expm1(pred)
 print(pred_exp )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def calculate_final_score(all_predictions, score_threshold):
final_scores = []
final_missed_boxes_nums = []
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_enboxes'].copy()
scores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores > score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<find_best_params>
submission.loc[:, 'count'] = pred_exp
submission
if is_TEST:
all_predictions = validation(net)
 best_final_score, best_score_threshold = 0, 0
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0], desc="OOF"):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<categorify>
submission.to_csv('submission.csv', index=False )
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= best_score_threshold].astype(np.int32)
 scores = scores[scores >= best_score_threshold]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
submission.to_csv('submission.csv', index=False )