Kaggle competition: Bike Sharing Demand
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from xgboost.sklearn import XGBRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import accuracy_score
from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
from keras.layers import Dropout
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from math import sqrt
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<load_pretrained>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def get_net(best_weight=None, checkpoint_path=None, validation=False, test=False):
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
if checkpoint_path != None:
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
elif best_weight != None:
net.load_state_dict(best_weight)
 gc.collect()
else:
raise RuntimeError()
if not test:
net = DetBenchTrain(net, config)
 else:
net = DetBenchPredict(net, config)
 if validation:
net.eval() ;
elif not test:
net.train()
return net.cuda()
net = get_net(None, ".. /input/wheatfold4/best-checkpoint-fold4-0.40127.bin", test = True )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
def add_dates(data , column , suffix='time_' , year = True , month = True , day = False ,dayofweek = True, hour = True , minute = False , second = False , date = False , time = False):
data['add_date_date_time'] = pd.to_datetime(data[column])
 if year == True:
data[suffix+'year']=data['add_date_date_time'].dt.year
if month == True:
data[suffix+'month']=data['add_date_date_time'].dt.month
if day == True:
data[suffix+'day']=data['add_date_date_time'].dt.day
if hour == True:
data[suffix+'hour']=data['add_date_date_time'].dt.hour
if minute == True:
data[suffix+'minute']=data['add_date_date_time'].dt.minute
if date == True:
data[suffix+'date']=data['add_date_date_time'].dt.date
if time == True:
data[suffix+'time']=data['add_date_date_time'].dt.time
if second == True:
data[suffix+'second']=data['add_date_date_time'].dt.second
if dayofweek == True:
data[suffix+'dayofweek']=data['add_date_date_time'].dt.dayofweek
data = data.drop(columns = ['add_date_date_time'] , axis =1)
 return data
train = add_dates(train , column = 'datetime')
 define_data(train , columns = True )
def process_det(index, det, score_threshold=0.33):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<define_variables>
def unique_count(data , columns = []):
for col in columns :
print('Unique Data Percentage in ',col)
 print(( data[col].value_counts() /data.shape[0])*100)
 print('
')
 unique_count(train , columns = ['season','weather','time_year', 'time_dayofweek'] )
IMAGE_SIZE = 1024<concatenate>
def hour_group(s):
if(( 0<=s)&(s<=6)) :
return 1
elif(( s==7)|(s==9)) :
return 2
elif(( s==8)|(s==16)|(s==19)) :
return 3
elif(( 10<=s)&(s<=15)) :
return 4
elif(( s==17)|(s==18)) :
return 5
elif(20<=s):
return 6
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
train['time_hour_group'] = train['time_hour'].apply(hour_group ).astype(str)
def make_tta_predictions(images, score_threshold=0.33):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()),
torch.tensor([1]*images.shape[0] ).float().cuda() ,
torch.tensor([[IMAGE_SIZE,IMAGE_SIZE]]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.43, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
train = new_col_categorical(train,columns=['season','weather','time_year', 'time_dayofweek' , 'time_month','time_hour_group'] , remove_original = False )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
train['weekend'] = train['time_dayofweek_5']+train['time_dayofweek_6']
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
test = add_dates(test , column = 'datetime')
 test['time_hour_group'] = test['time_hour'].apply(hour_group ).astype(str)
 test = new_col_categorical(test,columns=['season','weather','time_year', 'time_dayofweek' , 'time_month','time_hour_group'] , remove_original = False)
 test['weekend'] = test['time_dayofweek_5']+test['time_dayofweek_6']
test_x_new = test.drop(columns =['datetime',
'season', 'weather', 'time_year',
'time_dayofweek', 'time_month','time_hour_group'] , axis = 1 )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<define_variables>
scaler = MinMaxScaler()
train_x_new = scaler.fit_transform(train_x_new)
 train_y_new = np.log1p(train_y_new )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.25
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
SEED = 42
EPO = 15
WEIGHTS = '.. /input/wheatyolov5/best_wheat0.pt'
CONFIG = '.. /input/wheatyolov5/yolov5x.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True<set_options>
test_x_new_1 = scaler.transform(test_x_new)
 test_x_new_2 = scaler.fit_transform(test_x_new )
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
set_seed(SEED )<load_from_csv>
X_train , X_test , Y_train , Y_test = train_test_split(train_x_new , train_y_new , test_size =.15 , random_state = 65 )
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
valid_0_error =0
valid_1_error =0
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<concatenate>
def score_diff(valid_0_error , valid_1_error , valid_0_error_new , valid_1_error_new):
if valid_0_error == 0:
print('First Observaton')
 print('Train Error is : ',valid_0_error_new)
 print('Test Error is : ',valid_1_error_new)
 print('Diffence Between Train and Test is : ',(( valid_1_error_new-valid_0_error_new)/valid_0_error_new)*100 ,' %')
 else:
if valid_0_error_new > valid_0_error:
print('Train Error is : ',valid_0_error_new)
 print('Test Error is : ',valid_1_error_new)
 print('Train Error have Gone up by : ',(( valid_0_error_new-valid_0_error)/valid_0_error)*100,'%')
 print('Test Error have Gone up by : ',(( valid_1_error_new-valid_1_error)/valid_1_error)*100,'%')
 print('Diffence Between Train and Test is : ',(( valid_1_error_new-valid_0_error_new)/valid_0_error_new)*100 ,' %')
 print('Earlier Diffence Between Train and Test is : ',(( valid_1_error-valid_0_error)/valid_0_error)*100 ,' %')
 if valid_0_error_new < valid_0_error:
print('Train Error is : ',valid_0_error_new)
 print('Test Error is : ',valid_1_error_new)
 print('Train Error have Down up by : ',(( valid_0_error-valid_0_error_new)/valid_0_error_new)*100,'%')
 print('Test Error have Down up by : ',(( valid_1_error-valid_1_error_new)/valid_1_error_new)*100,'%')
 print('Diffence Between Train and Test is : ',(( valid_1_error_new-valid_0_error_new)/valid_0_error_new)*100 ,' %')
 print('Earlier Diffence Between Train and Test is : ',(( valid_1_error-valid_0_error)/valid_0_error)*100 ,' %')
 if valid_0_error_new == valid_0_error:
print('No Differnce in new Obseravtion')
 print('Train Error is : ',valid_0_error_new)
 print('Test Error is : ',valid_1_error_new)
 print('Diffence Between Train and Test is : ',(( valid_1_error_new-valid_0_error_new)/valid_0_error_new)*100 ,' %')
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
train_set = lgb.Dataset(X_train , label = Y_train)
 val_set = lgb.Dataset(X_test, label = Y_test )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
rank = np.argsort(scores)[::-1]
pred_boxes = pred_boxes[rank]
scores = scores[rank]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
%%time
params = {
"objective" : "regression",
"metric" : "mae",
"num_leaves" : 60,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"bagging_seed" : 0,
"num_threads" : 4,
"colsample_bytree" : 0.5,
'lambda_l2':9
}
model = lgb.train(params,
train_set = train_set,
num_boost_round=10000,
early_stopping_rounds=200,
verbose_eval=100,
valid_sets=[train_set,val_set] 
)
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
%%time
lgb_pred_test = model.predict(X_test, num_iteration=model.best_iteration)
 lgb_pred_train = model.predict(X_train, num_iteration=model.best_iteration)
 lgb_pred_normal = model.predict(test_x_new_1, num_iteration=model.best_iteration)
 lgb_pred_fit = model.predict(test_x_new_2, num_iteration=model.best_iteration )
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
valid_0_error_new = sqrt(mean_squared_error(np.array(Y_train),lgb_pred_train))
valid_1_error_new = sqrt(mean_squared_error(np.array(Y_test),lgb_pred_test))
score_diff(valid_0_error , valid_1_error , valid_0_error_new , valid_1_error_new)
 valid_0_error = valid_0_error_new
valid_1_error = valid_1_error_new
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
%%time
n_estimators=100
xgb = XGBRegressor(n_estimators=n_estimators,max_depth=4,learning_rate =0.01 , booster = 'gbtree')
 xgb.fit(X_train ,Y_train ,eval_set=[(X_train, Y_train),(X_test, Y_test)] , verbose = False)
 score = xgb.evals_result()
valid_0_error_new = np.amin(score['validation_0']['rmse'])
 valid_1_error_new = np.amin(score['validation_1']['rmse'])
 score_diff(valid_0_error , valid_1_error , valid_0_error_new , valid_1_error_new)
 valid_0_error = valid_0_error_new
valid_1_error = valid_1_error_new
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
%%time
model = RandomForestRegressor(random_state=65, n_estimators=200, min_samples_split=4)
 result = model.fit(X_train, Y_train )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
model.score(X_test, Y_test )
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
%%time
n_estimators=3000
xgb = XGBRegressor(n_estimators=n_estimators,max_depth=4,learning_rate =0.01 , booster = 'gbtree')
 xgb.fit(train_x_new , train_y_new ,eval_set=[(X_train, Y_train),(X_test, Y_test)] , verbose = False)
 pred_normal = xgb.predict(test_x_new_1)
 pred_fit = xgb.predict(test_x_new_2 )
!pip install tensorflow-gpu==2.0a0<import_modules>
%%time
model = RandomForestRegressor(random_state=65, n_estimators=n_estimators-2000)
 model.fit(train_x_new , train_y_new)
 rfr_pred_normal = model.predict(test_x_new_1)
 rfr_pred_fit = model.predict(test_x_new_2 )
print(tf.__version__ )<set_options>
%%time
pred_normal = np.expm1(pred_normal)
 pred_fit = np.expm1(pred_fit)
 rfr_pred_normal = np.expm1(rfr_pred_normal)
 rfr_pred_fit = np.expm1(rfr_pred_fit )
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
<define_variables>
%%time
NN_model = Sequential()
NN_model.add(Dense(128, kernel_initializer='normal',input_dim = train_x_new.shape[1], activation='relu'))
NN_model.add(Dense(256, kernel_initializer='normal',activation='relu'))
NN_model.add(Dropout(0.3))
NN_model.add(Dense(256, kernel_initializer='normal',activation='relu'))
NN_model.add(Dropout(0.3))
NN_model.add(Dense(256, kernel_initializer='normal',activation='relu'))
NN_model.add(Dropout(0.3))
NN_model.add(Dense(1, kernel_initializer='normal',activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary()
history = NN_model.fit(train_x_new,train_y_new, epochs=50, batch_size=64, verbose=1, validation_split=0.2)
 start_point = 150-100
r = range(start_point,150)
 plt.figure(figsize=(16,8))
plt.plot(history.history['loss'] ,'r' ,label ='Train')
 plt.plot(history.history['val_loss'] , 'g' , label = 'Test')
 plt.legend(fontsize='x-large')
 valid_0_error_new = history.history['loss'][-1]
valid_1_error_new = history.history['val_loss'][-1]
score_diff(valid_0_error , valid_1_error , valid_0_error_new , valid_1_error_new)
 valid_0_error = valid_0_error_new
valid_1_error = valid_1_error_new
tf.random.set_seed(42)
 datadir = ".. /input/"<choose_model_class>
ANN_pred_normal = NN_model.predict(test_x_new_1)
 ANN_pred_fit = NN_model.predict(test_x_new_2)
 ANN_pred_normal = np.expm1(ANN_pred_normal)
 ANN_pred_fit = np.expm1(ANN_pred_fit)
 ANN_pred_fit = ANN_pred_fit.reshape(6493)
 ANN_pred_normal = ANN_pred_normal.reshape(6493 )
<normalization><EOS> 
output = pd.DataFrame({'datetime': test.datetime,'count': pred_normal})
 output.to_csv('xgb_pred_normal.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': pred_fit})
 output.to_csv('xgb_pred_fit.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': rfr_pred_normal})
 output.to_csv('rfr_pred_normal.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': rfr_pred_fit})
 output.to_csv('rfr_pred_fit.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': ANN_pred_normal})
 output.to_csv('ANN_pred_normal.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': ANN_pred_fit})
 output.to_csv('ANN_pred_fit.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': lgb_pred_normal})
 output.to_csv('lgb_pred_normal.csv', index=False)
 output = pd.DataFrame({'datetime': test.datetime,'count': lgb_pred_fit})
 output.to_csv('lgb_pred_fit.csv', index=False )