Kaggle competition: Google Analytics Customer Revenue Prediction
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
train_df = pd.read_pickle('.. /input/gs-get-src-data/train_df', compression='gzip')
 print(train_df.shape)
 test_df = pd.read_pickle('.. /input/gs-get-src-data/test_df', compression='gzip')
 print(test_df.shape)
 train_df_u = pd.read_pickle('.. /input/gs-get-src-data/train_df_u', compression='gzip')
 print(train_df_u.shape)
 test_df_u = pd.read_pickle('.. /input/gs-get-src-data/test_df_u', compression='gzip')
 print(test_df_u.shape )
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
id_col = 'fullVisitorId'
cnt = train_df[id_col].value_counts()
ids = cnt.loc[cnt>1].index.values
print(ids.shape)
 df = train_df.loc[train_df[id_col].isin(ids)]
print(df.shape )
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
gdf = df.groupby(id_col)['target'].sum()
tids = gdf.loc[gdf>0].index.values
print(tids.shape)
 tdf = df.loc[df[id_col].isin(tids)]
print(tdf.shape )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
gdf = tdf.loc[:,[id_col,'target']]
gdf['target'] = gdf['target']>0
gdf = gdf.groupby(id_col)['target'].sum()
tids2 = gdf.loc[gdf>1].index.values
print(tids2.shape)
 tdf2 = tdf.loc[tdf[id_col].isin(tids2)]
print(tdf2.shape )
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
cols = list(tdf2.columns)
 cols = [col for col in cols if col != id_col] + [id_col]
tdf2[cols].loc[tdf2[id_col]==tids2[3]]
sys.path.insert(0, "/kaggle/input/weightedboxesfusion")
<define_variables>
tdf2.groupby([id_col,'visit_year'])['target'].sum().reset_index()
DATA_DIR = "/kaggle/input/global-wheat-detection"
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<load_from_csv>
cdf = tdf.loc[tdf[id_col].isin(cids)].groupby(id_col)['target'].sum()
print(cdf.shape)
 cdf
test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
test_df.shape<feature_engineering>
sub = pd.read_csv('.. /input/gs-lgb-uv-ensemble-1-1/lgb_uv_ensemble_1_1.csv')
 print(sub.shape)
 sub1 = sub.join(cdf, on=id_col)
 ind = sub1.target.notnull()
sub1.loc[ind, 'PredictedLogRevenue'] =(sub1.loc[ind, 'PredictedLogRevenue']+np.log1p(sub1.loc[ind, 'target'])) /2
submission = sub1.drop('target', axis=1)
 submission
<load_pretrained><EOS> 
submission.to_csv('test_all.csv', index=False, float_format='%.5f' )