Kaggle competition: Global Wheat Detection
def response_rate_by_group(df,x,y):
tmp = pd.crosstab(df[x],df[y])
 tmp['Sum'] = tmp.apply(np.sum,axis=1)
 tmp['Response_rate'] = tmp.loc[:,1]/tmp['Sum']
tmp = tmp.sort_values(['Response_rate'],ascending=False)
 print("It would be interesting to see if the amount percentual is higher or lower than 3.5% of total!")
 return(tmp)
 train['card3_high_rate_fraud'] = np.where(train['card3'].isin([185,119,144]),1,0)
 test['card3_high_rate_fraud'] = np.where(test['card3'].isin([185,119,144]),1,0)
 train['card5_high_rate_fraud'] = np.where(train['card5'].isin([137,147,141,223,138]),1,0)
 test['card5_high_rate_fraud'] = np.where(test['card5'].isin([137,147,141,223,138]),1,0)
 train['card5_137'] = np.where(train['card5'].isin([137]),1,0)
 test['card5_137'] = np.where(test['card5'].isin([137]),1,0)
 train['day'] = np.where(train['day'].isin([1,29,30,31]),1,0)
 test['day'] = np.where(test['day'].isin([1,29,30,31]),1,0)
 train['hour_from6_to10'] = np.where(train['hour'].isin([6,7,8,9,10]),1,0)
 test['hour_from6_to10'] = np.where(test['hour'].isin([6,7,8,9,10]),1,0)
 train['hour_from5_to11'] = np.where(train['hour'].isin([5,6,7,8,9,10,11]),1,0)
 test['hour_from5_to11'] = np.where(test['hour'].isin([5,6,7,8,9,10,11]),1,0)
 train['hour_from4_to12'] = np.where(train['hour'].isin([4,5,6,7,8,9,10,11,12]),1,0)
 test['hour_from4_to12'] = np.where(test['hour'].isin([4,5,6,7,8,9,10,11,12]),1,0)
 train['P_emaildomain_mail'] = np.where(train['P_emaildomain'].isin(['mail.com']),1,0)
 test['P_emaildomain_mail'] = np.where(test['P_emaildomain'].isin(['mail.com']),1,0)
 train['R_emaildomain_icloud'] = np.where(train['R_emaildomain'].isin(['icloud.com']),1,0)
 test['R_emaildomain_icloud'] = np.where(test['R_emaildomain'].isin(['icloud.com']),1,0)
 train['addr1_251'] = np.where(train['addr1'].isin([251]),1,0)
 test['addr1_251'] = np.where(test['addr1'].isin([251]),1,0)
 train['addr2_65'] = np.where(train['addr2'].isin([65]),1,0)
 test['addr2_65'] = np.where(test['addr2'].isin([65]),1,0)
 train['C1_0'] = np.where(train['C1'].isin([0]),1,0)
 test['C1_0'] = np.where(test['C1'].isin([0]),1,0)
 train['C1_19'] = np.where(train['C1'].isin([19]),1,0)
 test['C1_19'] = np.where(test['C1'].isin([19]),1,0 )<categorify>
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.5
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
EPO = 15
WEIGHTS = '.. /input/best-yolov5x-fold0-1pt/best_yolov5x_fold0.pt'
CONFIG = '.. /input/best-yolov5x-fold0pt/yolov5x.yaml'
DATA = '.. /input/best-yolov5x-fold0pt/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )
i_cols = ['card1','card2','card3','card5','uid', 'uid2', 'uid3','uid4', 'uid5']
for col in i_cols:
for agg_type in ['mean', 'std']:
new_col_name = col+'_TransactionAmt_'+agg_type
tmp = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 tmp = tmp.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(columns={agg_type: new_col_name})
 tmp.index = list(tmp[col])
 tmp = tmp[new_col_name].to_dict()
train[new_col_name] = train[col].map(tmp)
 test[new_col_name] = test[col].map(tmp)
 i_cols = ['card1','card2','card3','card5','uid', 'uid2', 'uid3','uid4', 'uid5','addr1','addr2']
for col in i_cols:
for agg_type in ['mean', 'std']:
new_col_name = col+'_'+'D15'+'_'+agg_type
tmp = pd.concat([train[[col, 'D15']], test[[col,'D15']]])
 tmp = tmp.groupby([col])['D15'].agg([agg_type] ).reset_index().rename(columns={agg_type: new_col_name})
 tmp.index = list(tmp[col])
 tmp = tmp[new_col_name].to_dict()
train[new_col_name] = train[col].map(tmp)
 test[new_col_name] = test[col].map(tmp)
 i_cols = ['card1','card4']
for col in i_cols:
for agg_type in ['mean', 'std']:
new_col_name = col+'_'+'id_02'+'_'+agg_type
tmp = pd.concat([train[[col, 'id_02']], test[[col,'id_02']]])
 tmp = tmp.groupby([col])['id_02'].agg([agg_type] ).reset_index().rename(columns={agg_type: new_col_name})
 tmp.index = list(tmp[col])
 tmp = tmp[new_col_name].to_dict()
train[new_col_name] = train[col].map(tmp)
 test[new_col_name] = test[col].map(tmp)
 i_cols = ['card1','card2','card3','card4','card5','uid', 'uid2', 'uid3','uid4', 'uid5']
for col in i_cols:
for agg_type in ['mean', 'std']:
new_col_name = col+'_'+'TransactionAmt_log'+'_'+agg_type
tmp = pd.concat([train[[col, 'TransactionAmt_log']], test[[col,'TransactionAmt_log']]])
 tmp = tmp.groupby([col])['TransactionAmt_log'].agg([agg_type] ).reset_index().rename(columns={agg_type: new_col_name})
 tmp.index = list(tmp[col])
 tmp = tmp[new_col_name].to_dict()
train[new_col_name] = train[col].map(tmp)
 test[new_col_name] = test[col].map(tmp )<categorify>
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std' )<categorify>
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )
for col in ['DT_M', 'DT_W', 'DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col + '_total'] = train[col].map(fq_encode)
 test[col + '_total'] = test[col].map(fq_encode)
 periods = ['DT_M', 'DT_W', 'DT_D']
i_cols = ['uid', 'uid2', 'uid3','uid4', 'uid5']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col, period]], test[[col, period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period + '_total']
test[new_column] /= test[period + '_total']<create_dataframe>
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )
def cal_woe(train, test, train_target,i_cols):
num_events = train_target.sum()
num_non_events = train_target.shape[0] - train_target.sum()
feature_list = []
feature_iv_list = []
for col in i_cols:
with timer('cope with %s' % col):
feature_list.append(col)
 woe_df = pd.DataFrame()
woe_df[col] = train[col]
woe_df['target'] = train_target
events_df = woe_df.groupby(col)['target'].sum().reset_index().rename(columns={'target' : 'events'})
 events_df['non_events'] = woe_df.groupby(col ).count().reset_index() ['target'] - events_df['events']
def cal_woe(x):
return np.log(((x['non_events']+0.5)/num_non_events)/(( x['events']+0.5)/num_events))
events_df['WOE_'+col] = events_df.apply(cal_woe, axis=1)
 def cal_iv(x):
return x['WOE_'+col]*(x['non_events'] / num_non_events - x['events'] / num_events)
 events_df['IV_'+col] = events_df.apply(cal_iv, axis=1)
 feature_iv = events_df['IV_'+col].sum()
feature_iv_list.append(feature_iv)
 events_df = events_df.drop(['events', 'non_events', 'IV_'+col], axis=1)
 train = train.merge(events_df, how='left', on=col)
 test = test.merge(events_df, how='left', on=col)
 return train, test
i_cols = [
'ProductCD',
'M4',
'id_12', 'id_15', 'id_16', 'id_28', 'id_29', 'id_35', 'id_36', 'id_37', 'id_38',
'DeviceType'
] 
<feature_engineering>
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )
prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']
for i, p in enumerate(prefix):
column_set = [x for x in train.columns.tolist() if x.startswith(prefix[i])]
print(column_set)
 train[p + "group_nan_sum"] = train[column_set].isnull().sum(axis=1)/ train[column_set].shape[1]
test[p + "group_nan_sum"] = test[column_set].isnull().sum(axis=1)/ test[column_set].shape[1]
numeric_cols = [x for x in column_set if train[x].dtype != object]
print(numeric_cols)
 if numeric_cols:
train[p + "group_sum"] = train[column_set].sum(axis=1)
 test[p + "group_sum"] = test[column_set].sum(axis=1)
 train[p + "group_mean"] = train[column_set].mean(axis=1)
 test[p + "group_mean"] = test[column_set].mean(axis=1)
 train[p + "group_0_count"] =(train[column_set] == 0 ).astype(int ).sum(axis=1)/(train[column_set].shape[1] - train[p + "group_nan_sum"])
 test[p + "group_0_count"] =(test[column_set] == 0 ).astype(int ).sum(axis=1)/(test[column_set].shape[1] - test[p + "group_nan_sum"] )<data_type_conversions>
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id))
<categorify>
if PSEUDO or VALIDATE:
convertTrainLabel()
for col in list(train):
if train[col].dtype=='O':
print(col)
 train[col] = train[col].fillna('unseen_before_label')
 test[col] = test[col].fillna('unseen_before_label')
 train[col] = train[col].astype(str)
 test[col] = test[col].astype(str)
 le = LabelEncoder()
le.fit(list(train[col])+list(test[col]))
train[col] = le.transform(train[col])
 test[col] = le.transform(test[col])
<data_type_conversions>
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<count_unique_values>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results
<save_to_csv><EOS> 
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()