Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<feature_engineering>
InteractiveShell.ast_node_interactivity = "all"
%matplotlib inline
nom_cols = [col for col in df_train.columns if 'nom' in col]
df_train['nom'] = df_train[nom_cols].apply('_'.join, axis=1)
 df_test['nom'] = df_test[nom_cols].apply('_'.join, axis=1)
 bin_cols = [col for col in df_train.columns if 'bin' in col]
df_train['bin'] = df_train[nom_cols].apply('_'.join, axis=1)
 df_test['bin'] = df_test[nom_cols].apply('_'.join, axis=1 )<train_model>
image_folder_path = "/kaggle/input/global-wheat-detection/train/"
chosen_image = cv2.imread(os.path.join(image_folder_path, "1ee6b9669.jpg")) [:,:,::-1]
plt.figure(figsize =(20,10))
plt.imshow(chosen_image )
classifier = LogisticRegressionCV(Cs=np.linspace(0.11, 0.14, num=100), max_iter=500, cv=10, n_jobs=-1)
 pl = ml_pipe(classifier)
 pl, pl_history, pl_accuracy = deploy_pipe(pl, df_train, y, df_train, y )<save_to_csv>
class InsectAugmentation(ImageOnlyTransform):

 def __init__(self, insects=2, dark_insect=False, always_apply=False, p=0.5):
super().__init__(always_apply, p)
 self.insects = insects
self.dark_insect = dark_insect
self.insects_folder = "/kaggle/input/bee-augmentation/"
def apply(self, image, **kwargs):

 n_insects = random.randint(1, self.insects)
 if not n_insects:
return image
height, width, _ = image.shape
insects_images = [im for im in os.listdir(self.insects_folder)if 'png' in im]
for _ in range(n_insects):
insect = cv2.cvtColor(cv2.imread(os.path.join(self.insects_folder, random.choice(insects_images))), cv2.COLOR_BGR2RGB)
 insect = cv2.flip(insect, random.choice([-1, 0, 1]))
insect = cv2.rotate(insect, random.choice([0, 1, 2]))
h_height, h_width, _ = insect.shape
roi_ho = random.randint(0, image.shape[0] - insect.shape[0])
 roi_wo = random.randint(0, image.shape[1] - insect.shape[1])
 roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(insect, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 if self.dark_insect:
img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 insect_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)
 else:
insect_fg = cv2.bitwise_and(insect, insect, mask=mask)
 dst = cv2.add(img_bg, insect_fg, dtype=cv2.CV_64F)
 image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return image
df_submit = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col=0)
 df_submit['target'] = pl.predict_proba(df_test)[:, 1]
df_submit.to_csv('cat_logreg_cv.csv' )<import_modules>
img = InsectAugmentation(insects=2, always_apply=True )(image=chosen_image)['image']
plt.figure(figsize =(20,10))
plt.imshow(img )
import pandas as pd
import numpy as np
from sklearn import preprocessing
import seaborn as sns
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import scipy
from sklearn import linear_model, datasets<load_from_csv>
chosen_image = cv2.imread(os.path.join(image_folder_path, "1ee6b9669.jpg")) [:,:,::-1]
img = InsectAugmentation(insects=2, dark_insect=True, always_apply=True )(image=chosen_image)['image']
plt.figure(figsize =(20,10))
plt.imshow(img )
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv", index_col='id')
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv", index_col = 'id' )<count_missing_values>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
print(train.isnull().sum().sum())
 print(test.isnull().sum().sum() )<count_unique_values>
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
test.nunique()<categorify>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
train['bin_4'] = train['bin_4'].map({'Y': 1, 'N': 0})
 train['bin_3'] = train['bin_3'].map({'T': 1, 'F': 0})
 train['ord_1'] = train['ord_1'].map({'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3,'Grandmaster': 4})
 train['ord_2'] = train['ord_2'].map({'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5} )<categorify>
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)
test['bin_4'] = test['bin_4'].map({'Y': 1, 'N': 0})
 test['bin_3'] = test['bin_3'].map({'T': 1, 'F': 0})
 test['ord_1'] = test['ord_1'].map({'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3,'Grandmaster': 4})
 test['ord_2'] = test['ord_2'].map({'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5} )<count_values>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d6')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/gwd-efficientdetd6-weights/fold0-efficientdetd6-PL-last-checkpoint1.bin' )
l3 = train[train['target'] == 1]['ord_3'].value_counts() / train['ord_3'].value_counts()
l4 = train[train['target'] == 1]['ord_4'].value_counts() / train['ord_4'].value_counts()
l5 = train[train['target'] == 1]['ord_5'].value_counts() / train['ord_5'].value_counts()<sort_values>
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
def sorted_ord(col_name,ratio):
s_ratio = ratio.sort_values()
keys = list(s_ratio.keys())
 train[col_name] = train[col_name].apply(lambda x : keys.index(x))
test[col_name] = test[col_name].apply(lambda x : keys.index(x))
sorted_ord('ord_3',l3)
 sorted_ord('ord_4',l4)
 sorted_ord('ord_5',l5 )<categorify>
def process_det(index, det, score_threshold=0.33):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
nom_col = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5','nom_6','nom_7','nom_8','nom_9','day','month']
nom_train = train[nom_col].astype(str)
 nom_test = test[nom_col].astype(str)
 ohc_test = pd.get_dummies(nom_test, sparse = True)
 ohc_train = pd.get_dummies(nom_train, sparse = True )<count_unique_values>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
unique_test = list(set(ohc_test.columns)- set(ohc_train.columns))
unique_train = list(set(ohc_train.columns)- set(ohc_test.columns))
print("Unique test columns: ", len(unique_test))
print("Unique Train columns: ",len(unique_train))
print(len(ohc_test.columns))
ohc_test_final = ohc_test.drop(unique_test, axis = 1)
 print(len(ohc_test_final.columns))
print(len(ohc_train.columns))
ohc_train_final = ohc_train.drop(unique_train, axis = 1)
 print(len(ohc_train_final.columns)) 
<drop_column>
def make_tta_predictions(images, score_threshold=0.33):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
dropped_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','day','month','target']
final_df = train.drop(dropped_cols, axis =1)
 target_train = train['target']
dropped_cols_test = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','day','month']
test_df = test.drop(dropped_cols_test, axis=1)
 df = final_df.to_sparse()
df1 = test_df.to_sparse()
df
df=df.to_coo()
df=df.tocsr()
df1
df1=df1.to_coo()
df1=df1.tocsr()
final_dataset = scipy.sparse.hstack([df,ohc_train_final])
 final_test = scipy.sparse.hstack([df1,ohc_test_final])
<train_on_grid>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
C = [0.12]
lr = LogisticRegression()
max_iter = [10000]
hyperparameters = dict(C=C, max_iter = max_iter)
 clf = GridSearchCV(lr, hyperparameters, cv=10, verbose=0)
 best_model = clf.fit(final_dataset,target_train )<predict_on_test>
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<save_to_csv><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()