Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<choose_model_class>
!pip install git+https://github.com/fastai/fastai@e85667cfae2e6873b1bb026195b5d09a74dfcff9
%load_ext autoreload
%autoreload 2
%matplotlib inline
def get_model_101(path,pretrained=False):
backbone = resnet_fpn_backbone('resnet101', pretrained=pretrained)
 model = FasterRCNN(backbone, num_classes=2)
 in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)
 model.load_state_dict(torch.load(path))
model.to(DEVICE)
 model.eval()
return model<load_pretrained>
from fastai.imports import *
from fastai.structured import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
model1 =get_model_101(".. /input/faster-cnn-final-model/fastercnn_resnet_101_145.pth")
 model2= get_model_101(".. /input/fastercnn-155/fastercnn_101_155.pth")
 model3= get_model_101(".. /input/faster-cnn-101/fastercnn_resnet_101_155_WightDecay.pth")
 faster_CNN_50_90_2=fasterrcnn_resnet50_fpn(".. /input/isfix-model/fastercnn_50_fix.pth")
 models=[model1,model3,model2,faster_CNN_50_90_2] 
<choose_model_class>
PATH = ".. /input/"
def display_all(df):
with pd.option_context("display.max_rows", 1000, "display.max_columns", 1000):
display(df)
 def load_df(csv_path, nrows=None):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
types = {
"fullVisitorId": "str",
"channelGrouping": "str",
"date": "str",
"socialEngagementType": "str",
"visitId": "int32",
"visitNumber": "int8",
"visitStartTime": "int32",
}
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype=types,
nrows=nrows)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df
def make_ensemble_predictions(images):
images = list(image.to(DEVICE)for image in images)
 result = []
for model in models:
with torch.no_grad() :
outputs = model(images)
 result.append(outputs)
 del model
gc.collect()
torch.cuda.empty_cache()
return result
def run_wbf_ensemble(predictions, image_index, image_size=1024, iou_thr=0.34, skip_box_thr=0.43, weights=None):
boxes = [prediction[image_index]['boxes'].data.cpu().numpy() /(image_size-1)for prediction in predictions]
scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0])for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<concatenate>
def split_vals(a,n):
return a[:n].copy() , a[n:].copy()
def rmse(x,y):
return math.sqrt(((x-y)**2 ).mean())
 def print_score(m):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
def get_test_transforms() :
return A.Compose([
ToTensorV2(p=1.0)
 ], p=1.0 )<create_dataframe>
df_raw = load_df(f'{PATH}train_v2.csv', nrows=500_000 )
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=1,
drop_last=False,
collate_fn=collate_fn
)<define_variables>
const_cols = [c for c in df_raw.columns if df_raw[c].nunique(dropna=False)==1]
df_raw.drop(columns=const_cols + ["customDimensions", "hits"], inplace=True )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
df_test = load_df(f'{PATH}test_v2.csv' )
class BaseWheatTTA:

 image_size = 1024
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
df_raw.drop(columns=['trafficSource.campaignCode'], inplace=True )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<predict_on_test>
df_test.drop(columns=const_cols + ["customDimensions", "hits"], inplace=True )
def make_tta_predictions(images, score_threshold=0.57):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = make_ensemble_predictions(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes, scores, labels = run_wbf_ensemble(outputs, image_index=i)
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
df_raw['totals.transactionRevenue'].fillna(0, inplace=True)
 df_raw['totals.transactionRevenue'] = df_raw['totals.transactionRevenue'].astype(float)
 df_raw['totals.transactionRevenue'] = np.log1p(df_raw['totals.transactionRevenue'] )
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.6, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<categorify>
df_raw["visitStartTime"] = pd.to_datetime(df_raw["visitStartTime"], infer_datetime_format=True, unit="s")
 df_raw["date"] = pd.to_datetime(df_raw["date"], infer_datetime_format=True, format="%Y%m%d")
 add_datepart(df_raw, 'date')
 add_datepart(df_raw, 'visitStartTime' )
results = []
list_box=[]
list_score=[]
list_image_id=[]
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
list_score.append(scores)
 list_box.append(boxes)
 list_image_id.append(image_id)
 result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<create_dataframe>
df_raw['totals.totalTransactionRevenue'].fillna(0, inplace=True)
 df_raw['totals.totalTransactionRevenue'] = df_raw['totals.totalTransactionRevenue'].astype('int32')
 df_raw['totals.transactions'].fillna(0, inplace=True)
 df_raw['totals.transactions'] = df_raw['totals.transactions'].astype('int8' )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df<save_to_csv>
df_test['totals.transactionRevenue'].fillna(0, inplace=True)
 df_test['totals.transactionRevenue'] = df_test['totals.transactionRevenue'].astype(float)
 df_test['totals.transactionRevenue'] = np.log1p(df_test['totals.transactionRevenue'] )
test_df.to_csv('submission.csv', index=False )<define_variables>
df_test["visitStartTime"] = pd.to_datetime(df_test["visitStartTime"], infer_datetime_format=True, unit="s")
 df_test["date"] = pd.to_datetime(df_test["date"], infer_datetime_format=True, format="%Y%m%d")
 add_datepart(df_test, 'date')
 add_datepart(df_test, 'visitStartTime' )
weights = [".. /input/40efolds/best48fold0.pt", ".. /input/40efolds/best48fold1.pt", ".. /input/40efolds/best48fold2.pt", ".. /input/40efolds/best48fold3.pt", ".. /input/40efolds/best40fold4.pt"]
print(len(weights))
source = ".. /input/global-wheat-detection/test"
imgsz = 1024
conf_t = 0.5
iou_t = 0.8
built_in_tta = True
device = torch_utils.select_device("0" if torch.cuda.is_available() else "")
 half = device.type != 'cpu'<define_search_space>
train_cats(df_raw)
 train_cats(df_test )
skip_box_thr = 0.43
iou_thr = 0.6
score_thr = 0.25
ws = [0.15, 0.15, 0.27, 0.15, 0.28]
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.34, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores<normalization>
df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue' )
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, conf_t, iou_t, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<find_best_params>
m = RandomForestRegressor(n_jobs=-1)
 m.fit(df_trn, y_trn)
 m.score(df_trn, y_trn )
def detect() :
models = []
for w in weights if isinstance(weights, list)else [weights]:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=imgsz)
 img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 for model in models:
if half:
model.half()
_ = model(img.half() if half else img)if device.type != 'cpu' else None
all_paths, all_bboxes, all_confs = [], [], []
for p, img, img0, _ in dataset:
print()
img = img.transpose(1,2,0)
 all_paths.append(p)
 m_box, m_score = [], []
for model in models:
enboxes = []
enscores = []
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=True)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 indices = scores >= score_thr
boxes = boxes[indices]
scores = scores[indices]
m_box.append(boxes)
 m_score.append(scores)
 all_bboxes.append(m_box)
 all_confs.append(m_score)
 return all_paths, all_bboxes, all_confs<find_best_params>
train_required_ratio = 0.8
n_trn = int(len(df_trn)* train_required_ratio)
 X_train, X_valid = split_vals(df_trn, n_trn)
 y_train, y_valid = split_vals(y_trn, n_trn)
 X_train.shape, X_valid.shape
with torch.no_grad() :
res = detect()
paths,all_boxes,all_confs = res 
<define_variables>
m = RandomForestRegressor(n_estimators=10, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )
resultsYOLO =dict()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 bts = []
sts = []
for row in range(len(paths)) :
image_id = paths[row].split("/")[-1].split(".")[0]
boxes, scores = run_wbf(all_boxes[row],all_confs[row], skip_box_thr = skip_box_thr, iou_thr = iou_thr, weights = None)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 resultsYOLO[image_id] = [boxes, scores]
!rm -rf * 
<import_modules>
preds = np.stack([t.predict(X_valid)for t in m.estimators_])
 preds[:,0], np.mean(preds[:,0]), y_valid[0]
preds.shape
sys.path.insert(0, "/kaggle/input/weightedboxesfusion")
<define_variables>
plt.plot([metrics.r2_score(y_valid, np.mean(preds[:i+1], axis=0)) for i in range(10)]);
DATA_DIR = "/kaggle/input/global-wheat-detection"
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<load_from_csv>
m = RandomForestRegressor(n_estimators=40, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
test_df.shape<feature_engineering>
set_rf_samples(50_000 )
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id 
<load_pretrained>
m = RandomForestRegressor(n_estimators=40, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )
def fasterrcnn_resnet50_fpn(path,pretrained_backbone=False):
backbone = resnet_fpn_backbone('resnet50', pretrained_backbone)
 model = FasterRCNN(backbone, 2)
 model.load_state_dict(torch.load(path))
model.to(DEVICE)
 model.eval()
return model<choose_model_class>
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
def get_model_101(path,pretrained=False):
backbone = resnet_fpn_backbone('resnet101', pretrained=pretrained)
 model = FasterRCNN(backbone, num_classes=2)
 in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)
 model.load_state_dict(torch.load(path))
model.to(DEVICE)
 model.eval()
return model<load_pretrained>
df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue')
 train_required_ratio = 0.8
n_trn = int(len(df_trn)* train_required_ratio)
 X_train, X_valid = split_vals(df_trn, n_trn)
 y_train, y_valid = split_vals(y_trn, n_trn)
 X_test, y_test, _ = proc_df(df_test, 'totals.transactionRevenue', nas)
 X_train.shape, X_valid.shape, X_test.shape
model =get_model_101(".. /input/faster-cnn-final-model/fastercnn_resnet_101_145.pth")
 model2 =get_model_101(".. /input/faster-cnn-final-model/fastercnn_resnet_101_145.pth")
 model3=get_model_101(".. /input/faster-cnn-101-155-fix/fastercnn_resnet_101_fix_155.pth")
 model4= get_model_101(".. /input/fastercnn-155/fastercnn_101_155.pth")
 model5= get_model_101(".. /input/faster-cnn-125/fastercnn_resnet_101_125.pth")
 faster_CNN_50_90=fasterrcnn_resnet50_fpn(".. /input/isfix-model/fastercnn_50_fix.pth")
 faster_CNN_50_90_2=fasterrcnn_resnet50_fpn(".. /input/isfix-model/fastercnn_50_fix.pth")
 faster_CNN_50_105=fasterrcnn_resnet50_fpn(".. /input/faster-50-fix-115/fastercnn_50_fix_115.pth")
 models=[model,model3,model4,model5,faster_CNN_50_105,faster_CNN_50_90,faster_CNN_50_90_2
,model2
]<choose_model_class>
m = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train )
def make_ensemble_predictions(images):
images = list(image.to(DEVICE)for image in images)
 result = []
for model in models:
with torch.no_grad() :
outputs = model(images)
 result.append(outputs)
 del model
gc.collect()
torch.cuda.empty_cache()
return result
def run_wbf_ensemble(predictions, image_index, image_size=1024, iou_thr=0.45, skip_box_thr=0.43, weights=None):
boxes = [prediction[image_index]['boxes'].data.cpu().numpy() /(image_size-1)for prediction in predictions]
scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0])for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<concatenate>
print_score(m )
def get_test_transforms() :
return A.Compose([
ToTensorV2(p=1.0)
 ], p=1.0 )<create_dataframe>
predictions = m.predict(X_test )
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=1,
drop_last=False,
collate_fn=collate_fn
)<define_variables>
set_rf_samples(50000 )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
raw_train, raw_valid = split_vals(df_raw, n_trn )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<predict_on_test>
fi = rf_feat_importance(m, df_trn); fi[:30]
def make_tta_predictions(images, score_threshold=0.57):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = make_ensemble_predictions(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes, scores, labels = run_wbf_ensemble(outputs, image_index=i)
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
fi[fi.imp>0.0001]
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.4, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
to_keep = fi[fi.imp>0.0001].cols; len(to_keep )
resultsFASTER = {}
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
resultsFASTER[image_id] = [boxes, scores] 
<compute_train_metric>
df_keep = df_trn[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.6, skip_box_thr=0.43, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores<categorify>
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5,
n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
pred_string = []
bts, sts, ids = [], [], []
results = dict()
for image_id, faster in resultsFASTER.items() :
yolo = resultsYOLO[image_id]
scores_ = [faster[1], yolo[1],yolo[1]]
boxes_ = [faster[0], yolo[0],yolo[0]]
boxes, scores = run_wbf(boxes_, scores_, iou_thr = 0.45, skip_box_thr=0.01)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 indices = scores >= 0.3
boxes = boxes[indices]
scores = scores[indices]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id, 'PredictionString': format_prediction_string(boxes, scores)}
pred_string.append(result)
 ids.append(image_id)
 bts.append(boxes)
 sts.append(scores)
 test_df = pd.DataFrame(pred_string, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<define_variables>
df_trn2, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue', max_n_cat=7)
 X_train, X_valid = split_vals(df_trn2, n_trn)
 m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.25
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
SEED = 42
EPO = 5
WEIGHTS = '.. /input/xiaomai-yolov5/bestv4.pt'
CONFIG = '.. /input/configyolo5/yolov5x.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) <11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True<set_options>
from scipy.cluster import hierarchy as hc
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
set_seed(SEED )<load_from_csv>
df_keep = df_trn[to_keep].copy()
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
def get_oob(df):
m = RandomForestRegressor(n_estimators=30, min_samples_leaf=5, max_features=0.6, n_jobs=-1, oob_score=True)
 x, _ = split_vals(df, n_trn)
 m.fit(x, y_train)
 return m.oob_score_
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<concatenate>
get_oob(df_keep )
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
for c in('totals.transactions', 'totals.totalTransactionRevenue'):
print(c, get_oob(df_keep.drop(c, axis=1)) )
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
rank = np.argsort(scores)[::-1]
pred_boxes = pred_boxes[rank]
scores = scores[rank]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
to_drop = ['totals.transactions']
get_oob(df_keep.drop(to_drop, axis=1))
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
df_keep.drop(to_drop, axis=1, inplace=True)
 X_train, X_valid = split_vals(df_keep, n_trn )
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
np.save('tmp/keep_cols.npy', np.array(df_keep.columns))
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
keep_cols = np.load('tmp/keep_cols.npy')
 df_keep = df_trn[keep_cols]
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
!pip install treeinterpreter
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
from treeinterpreter import treeinterpreter as ti
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<install_modules>
df_train, df_valid = split_vals(df_raw[df_keep.columns], n_trn )
!pip install --no-deps '.. /input/timm0130/timm-0.1.30-py3-none-any.whl' > /dev/null<set_options>
prediction, bias, contributions = ti.predict(m, row )
%matplotlib inline
VerticalFlip, HorizontalFlip, IAASharpen,
OneOf, Compose , BboxParams, Resize, HueSaturationValue
,RandomBrightnessContrast, ToGray , Cutout , RandomSizedCrop )<import_modules>
idxs = np.argsort(contributions[0] )
from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain , DetBenchPredict
from effdet.efficientdet import HeadNet<define_variables>
DIR_PATH = '.. /input/global-wheat-detection/'
dir = glob.glob(os.path.join(DIR_PATH , '*'))
dir.sort(reverse=True)
 print(dir)
 train_paths = glob.glob(os.path.join(dir[1] , '*'))
test_paths = glob.glob(os.path.join(dir[2] , '*'))<feature_engineering>
df_ext = df_keep.copy()
df_ext['is_valid'] = 1
df_ext.is_valid[:n_trn] = 0
x, y, nas = proc_df(df_ext, 'is_valid' )
df = pd.read_csv(dir[0])
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df<feature_engineering>
m = RandomForestClassifier(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(x, y);
m.oob_score_
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = df[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = df[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds<groupby>
fi = rf_feat_importance(m, x); fi[:10]
df_folds.groupby('fold')['bbox_count'].mean()<train_model>
feats=['totals.timeOnSite']
image_id = '8425a537b.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image without bounding box" )<train_model>
x.drop(feats, axis=1, inplace=True )
image_id = 'b3c96d5ad.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image with bounding box" )<normalization>
m = RandomForestClassifier(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(x, y);
m.oob_score_
images_after_augmentation(image, boxes, Blur(blur_limit=7 ,p=1))<normalization>
fi = rf_feat_importance(m, x); fi[:10]
images_after_augmentation(image, boxes, HorizontalFlip(p=1))<feature_engineering>
set_rf_samples(50000 )
class WheatDataset(Dataset):
def __init__(self , dataframe , image_ids, transforms = None):
super().__init__()
self.image_ids = image_ids
self.dataframe = dataframe
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['labels'] = torch.stack(sample['labels'])
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[1]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.dataframe[self.dataframe['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
area =(boxes[:, 2] * boxes[:, 3])
 boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes<concatenate>
feats=['totals.timeOnSite', 'geoNetwork.country', 'totals.hits', 'totals.pageviews']
def collate_fn(batch):
return tuple(zip(*batch))<create_dataframe>
X_train, X_valid = split_vals(df_keep, n_trn)
 m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
bs = 2
fold_number = 0
train_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] != fold_number].index.values , transforms=transforms_train)
 valid_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] == fold_number].index.values , transforms=transforms_valid)
 train_loader = DataLoader(train_set,batch_size = bs,sampler=RandomSampler(train_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=2)
 valid_loader = DataLoader(valid_set,batch_size = bs ,sampler=SequentialSampler(valid_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=2)
 images , targets , path_images = next(iter(train_loader))
img = visualizeTarget(images[0],targets[0])
 visualize(Example_one_image_from_dataloader = img )<choose_model_class>
for f in feats:
df_subs = df_keep.drop(f, axis=1)
 X_train, X_valid = split_vals(df_subs, n_trn)
 m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print(f)
 print_score(m )
class Training:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.best_calc_loss = 10**5
self.model = model
self.device = device
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 def train_loop(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
print(f'
{timestamp}
LR: {lr}')
 t = time.time()
calc_loss = self.train_one_epoch(train_loader)
 print(f'Train.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save_model(f'{self.base_dir}/last-epoch.bin')
 t = time.time()
calc_loss = self.valid_one_epoch(validation_loader)
 print(f'Val.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if calc_loss.avg < self.best_calc_loss:
self.best_calc_loss = calc_loss.avg
self.model.eval()
self.save_model(f'{self.base_dir}/best-model-{str(self.epoch ).zfill(3)}epoch.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=calc_loss.avg)
 self.epoch += 1
def train_one_epoch(self, train_loader):
self.model.train()
calc_loss = Calculator()
t = time.time()
target_res = {}
for images, targets, image_ids in tqdm_notebook(train_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res['bbox'] = boxes
target_res['cls'] = labels
self.optimizer.zero_grad()
outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 loss.backward()
self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return calc_loss
def valid_one_epoch(self, val_loader):
self.model.eval()
calc_loss = Calculator()
t = time.time()
with torch.no_grad() :
for images, targets, image_ids in tqdm_notebook(val_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res = {}
target_res['bbox'] = boxes
target_res['cls'] = labels
target_res["img_scale"] = torch.tensor([1.0] * batch_size, dtype=torch.float ).to(self.device)
 target_res["img_size"] = torch.tensor([images[0].shape[-2:]] * batch_size, dtype=torch.float ).to(self.device)
 outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 return calc_loss
def save_model(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_calc_loss': self.best_calc_loss,
'epoch': self.epoch,
}, path)
 def load_model(self, path):
model = torch.load(path)
 self.model.load_state_dict(model['model_state_dict'])
 self.optimizer.load_state_dict(model['optimizer_state_dict'])
 self.scheduler.load_state_dict(model['scheduler_state_dict'])
 self.best_calc_loss = model['best_calc_loss']
self.epoch = model['epoch'] + 1<choose_model_class>
df_subs = df_keep.drop(['geoNetwork.country'], axis=1)
 X_train, X_valid = split_vals(df_subs, n_trn)
 m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
def get_model(num_classes = 1):
config = get_efficientdet_config('tf_efficientdet_d5')
 model = EfficientDet(config, pretrained_backbone=False)
 checkpoint = torch.load('/content/drive/My Drive/Global Wheat Detection/efficientdet_d7/efficientdet_d5-ef44aea8.pth')
 model.load_state_dict(checkpoint)
 config.num_classes = num_classes
config.image_size = 512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
return DetBenchTrain(model, config )<choose_model_class>
np.save('tmp/subs_cols.npy', np.array(df_subs.columns))
class GlobalParametersTrain:
lr = 0.0002
n_epochs = 40
folder = '/content/drive/My Drive/Global Wheat Detection/efficientdet_d5'
verbose = True
verbose_step = 10
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(mode='min',factor=0.5,patience=1,verbose=False, threshold=0.0001,threshold_mode='abs',cooldown=0, min_lr=1e-8,eps=1e-08 )<choose_model_class>
m = RandomForestRegressor(n_estimators=160, max_features=0.5, n_jobs=-1, oob_score=True)
 %time m.fit(X_train, y_train)
 print_score(m )
def load_model(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 model = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchPredict(model, config)
 model.eval() ;
return model.cuda()
model = load_model('.. /input/efficientdetd5model17e/efficient-best-017epoch.bin' )<predict_on_test>
X_test = X_test.drop(set(df_raw.columns ).difference(set(df_subs.columns)) - {'totals.transactionRevenue'}, axis=1);
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def make_predictions(images , score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
outputs = model(images , torch.tensor([1.0] * images.shape[0], dtype=torch.float ).to(device), torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes],
})
 return [predictions]<compute_train_metric>
predictions = m.predict(X_test )
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
df_actual_test = df_test.copy()
for j,(images, targets , image_ids)in enumerate(valid_loader):
break
predictions = make_predictions(images)
 i = 0
sample = images[i].permute(1,2,0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=511 )<categorify>
actual_predicted_revenue = predictions
df_actual_test["predicted"] = actual_predicted_revenue
df_actual_test = df_actual_test[["fullVisitorId" , "predicted"]]
df_actual_test["fullVisitorId"] = df_actual_test.fullVisitorId.astype('str')
 df_actual_test["predicted"] = df_actual_test.predicted.astype(np.float)
 df_actual_test.index = df_actual_test.fullVisitorId
df_actual_test = df_actual_test.drop("fullVisitorId",axis=1 )
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
df_actual_test["predicted"].value_counts().iloc[:5]
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None], [TTAVerticalFlip() , None],[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
df_submission_test = pd.read_csv(filepath_or_buffer=".. /input/sample_submission_v2.csv",index_col="fullVisitorId")
 df_submission_test.shape
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda() , torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
'labels': labels[indexes],
})
 predictions.append(result)
 return predictions<categorify>
final_df = final_df[~final_df.index.duplicated(keep='first')]
final_df = final_df.rename(index=str, columns={"predicted": "PredictedLogRevenue"} )
test_transforms = Compose([
Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
final_df.PredictedLogRevenue.value_counts(bins=3 )
class TestDataset(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[2]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self):
return self.image_ids.shape[0]<create_dataframe>
final_df = final_df.fillna(0); final_df.iloc[262153]
<define_variables><EOS> 
final_df.to_csv("sub.csv" )