Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<install_modules>
import numpy as np
import pandas as pd
import os
from tqdm.auto import tqdm
import shutil as sh
!rm -r /opt/conda/lib/python3.6/site-packages/lightgbm
!git clone --recursive https://github.com/Microsoft/LightGBM
!apt-get install -y -qq libboost-all-dev<install_modules>
!cp -r.. /input/yolov5-pseudo-labeling/* .
%%bash
cd LightGBM
rm -r build
mkdir build
cd build
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/..
make -j$(nproc )<install_modules>
!pip install --no-deps '.. /input/weightedboxesfusion/' > /dev/null
!cd LightGBM/python-package/;python3 setup.py install --precompile<install_modules>
R_fold = 1
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*R_fold//5:len(index)*(R_fold+1)//5]
for name,mini in tqdm(df.groupby('image_id')) :
if name in val_index:
path2save = 'val2017/'
else:
path2save = 'train2017/'
if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)) :
os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))
sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))
!mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
!rm -r LightGBM
!pip install -q 'pandas==0.25' --force-reinstall
!pip install GPyOpt<import_modules>
def run_wbf(boxes, scores, image_size=1023, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(im0, imgsz, model, device, conf_thres, iou_thres):
img = letterbox(im0, new_shape=imgsz)[0]
img = img[:, :, ::-1].transpose(2, 0, 1)
 img = np.ascontiguousarray(img)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=False)[0]
pred = non_max_suppression(pred, conf_thres, iou_thres)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )
print("GPyOpt version:", GPyOpt.__version__)
 print("LGBM version:", lgb.__version__)
 print("Pandas version:", pd.__version__ )<set_options>
def makePseudolabel(weights = '.. /input/yolov5pth/weightsbest_yolov5x_fold3.pt'):
source = '.. /input/global-wheat-detection/test/'
imgsz = 1024
conf_thres = 0.5
iou_thres = 0.6
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for name in tqdm(imagenames):
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
if im01.shape[0]!=1024 or im01.shape[1]!=1024:
continue
assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.6, skip_box_thr=0.43)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.1].astype(np.int32)
 scores = scores[scores >=float(0.1)]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id))
notebookstart = time.time()
warnings.filterwarnings("ignore")
 seed = 24
np.random.seed(seed)
 pd.set_option('display.max_columns', 500)
 pd.options.display.max_rows = 999
pd.set_option('max_colwidth', 500 )<define_variables>
print("Define DF Schema.. ")
 target_var = 'isFraud'
schema = {
"TransactionDT": "int32",
"TransactionAmt": "float32",
"ProductCD": "object",
"card1": "int16",
"card2": "float32",
"card3": "float32",
"card4": "object",
"card5": "float32",
"card6": "object",
"addr1": "float32",
"addr2": "float32",
"dist1": "float32",
"dist2": "float32",
"P_emaildomain": "object",
"R_emaildomain": "object",
"C1": "float32",
"C2": "float32",
"C3": "float32",
"C4": "float32",
"C5": "float32",
"C6": "float32",
"C7": "float32",
"C8": "float32",
"C9": "float32",
"C10": "float32",
"C11": "float32",
"C12": "float32",
"C13": "float32",
"C14": "float32",
"D1": "float32",
"D2": "float32",
"D3": "float32",
"D4": "float32",
"D5": "float32",
"D6": "float32",
"D7": "float32",
"D8": "float32",
"D9": "float32",
"D10": "float32",
"D11": "float32",
"D12": "float32",
"D13": "float32",
"D14": "float32",
"D15": "float32",
"M1": "object",
"M2": "object",
"M3": "object",
"M4": "object",
"M5": "object",
"M6": "object",
"M7": "object",
"M8": "object",
"M9": "object",
"V1": "float32",
"V2": "float32",
"V3": "float32",
"V4": "float32",
"V5": "float32",
"V6": "float32",
"V7": "float32",
"V8": "float32",
"V9": "float32",
"V10": "float32",
"V11": "float32",
"V12": "float32",
"V13": "float32",
"V14": "float32",
"V15": "float32",
"V16": "float32",
"V17": "float32",
"V18": "float32",
"V19": "float32",
"V20": "float32",
"V21": "float32",
"V22": "float32",
"V23": "float32",
"V24": "float32",
"V25": "float32",
"V26": "float32",
"V27": "float32",
"V28": "float32",
"V29": "float32",
"V30": "float32",
"V31": "float32",
"V32": "float32",
"V33": "float32",
"V34": "float32",
"V35": "float32",
"V36": "float32",
"V37": "float32",
"V38": "float32",
"V39": "float32",
"V40": "float32",
"V41": "float32",
"V42": "float32",
"V43": "float32",
"V44": "float32",
"V45": "float32",
"V46": "float32",
"V47": "float32",
"V48": "float32",
"V49": "float32",
"V50": "float32",
"V51": "float32",
"V52": "float32",
"V53": "float32",
"V54": "float32",
"V55": "float32",
"V56": "float32",
"V57": "float32",
"V58": "float32",
"V59": "float32",
"V60": "float32",
"V61": "float32",
"V62": "float32",
"V63": "float32",
"V64": "float32",
"V65": "float32",
"V66": "float32",
"V67": "float32",
"V68": "float32",
"V69": "float32",
"V70": "float32",
"V71": "float32",
"V72": "float32",
"V73": "float32",
"V74": "float32",
"V75": "float32",
"V76": "float32",
"V77": "float32",
"V78": "float32",
"V79": "float32",
"V80": "float32",
"V81": "float32",
"V82": "float32",
"V83": "float32",
"V84": "float32",
"V85": "float32",
"V86": "float32",
"V87": "float32",
"V88": "float32",
"V89": "float32",
"V90": "float32",
"V91": "float32",
"V92": "float32",
"V93": "float32",
"V94": "float32",
"V95": "float32",
"V96": "float32",
"V97": "float32",
"V98": "float32",
"V99": "float32",
"V100": "float32",
"V101": "float32",
"V102": "float32",
"V103": "float32",
"V104": "float32",
"V105": "float32",
"V106": "float32",
"V107": "float32",
"V108": "float32",
"V109": "float32",
"V110": "float32",
"V111": "float32",
"V112": "float32",
"V113": "float32",
"V114": "float32",
"V115": "float32",
"V116": "float32",
"V117": "float32",
"V118": "float32",
"V119": "float32",
"V120": "float32",
"V121": "float32",
"V122": "float32",
"V123": "float32",
"V124": "float32",
"V125": "float32",
"V126": "float32",
"V127": "float32",
"V128": "float32",
"V129": "float32",
"V130": "float32",
"V131": "float32",
"V132": "float32",
"V133": "float32",
"V134": "float32",
"V135": "float32",
"V136": "float32",
"V137": "float32",
"V138": "float32",
"V139": "float32",
"V140": "float32",
"V141": "float32",
"V142": "float32",
"V143": "float32",
"V144": "float32",
"V145": "float32",
"V146": "float32",
"V147": "float32",
"V148": "float32",
"V149": "float32",
"V150": "float32",
"V151": "float32",
"V152": "float32",
"V153": "float32",
"V154": "float32",
"V155": "float32",
"V156": "float32",
"V157": "float32",
"V158": "float32",
"V159": "float32",
"V160": "float32",
"V161": "float32",
"V162": "float32",
"V163": "float32",
"V164": "float32",
"V165": "float32",
"V166": "float32",
"V167": "float32",
"V168": "float32",
"V169": "float32",
"V170": "float32",
"V171": "float32",
"V172": "float32",
"V173": "float32",
"V174": "float32",
"V175": "float32",
"V176": "float32",
"V177": "float32",
"V178": "float32",
"V179": "float32",
"V180": "float32",
"V181": "float32",
"V182": "float32",
"V183": "float32",
"V184": "float32",
"V185": "float32",
"V186": "float32",
"V187": "float32",
"V188": "float32",
"V189": "float32",
"V190": "float32",
"V191": "float32",
"V192": "float32",
"V193": "float32",
"V194": "float32",
"V195": "float32",
"V196": "float32",
"V197": "float32",
"V198": "float32",
"V199": "float32",
"V200": "float32",
"V201": "float32",
"V202": "float32",
"V203": "float32",
"V204": "float32",
"V205": "float32",
"V206": "float32",
"V207": "float32",
"V208": "float32",
"V209": "float32",
"V210": "float32",
"V211": "float32",
"V212": "float32",
"V213": "float32",
"V214": "float32",
"V215": "float32",
"V216": "float32",
"V217": "float32",
"V218": "float32",
"V219": "float32",
"V220": "float32",
"V221": "float32",
"V222": "float32",
"V223": "float32",
"V224": "float32",
"V225": "float32",
"V226": "float32",
"V227": "float32",
"V228": "float32",
"V229": "float32",
"V230": "float32",
"V231": "float32",
"V232": "float32",
"V233": "float32",
"V234": "float32",
"V235": "float32",
"V236": "float32",
"V237": "float32",
"V238": "float32",
"V239": "float32",
"V240": "float32",
"V241": "float32",
"V242": "float32",
"V243": "float32",
"V244": "float32",
"V245": "float32",
"V246": "float32",
"V247": "float32",
"V248": "float32",
"V249": "float32",
"V250": "float32",
"V251": "float32",
"V252": "float32",
"V253": "float32",
"V254": "float32",
"V255": "float32",
"V256": "float32",
"V257": "float32",
"V258": "float32",
"V259": "float32",
"V260": "float32",
"V261": "float32",
"V262": "float32",
"V263": "float32",
"V264": "float32",
"V265": "float32",
"V266": "float32",
"V267": "float32",
"V268": "float32",
"V269": "float32",
"V270": "float32",
"V271": "float32",
"V272": "float32",
"V273": "float32",
"V274": "float32",
"V275": "float32",
"V276": "float32",
"V277": "float32",
"V278": "float32",
"V279": "float32",
"V280": "float32",
"V281": "float32",
"V282": "float32",
"V283": "float32",
"V284": "float32",
"V285": "float32",
"V286": "float32",
"V287": "float32",
"V288": "float32",
"V289": "float32",
"V290": "float32",
"V291": "float32",
"V292": "float32",
"V293": "float32",
"V294": "float32",
"V295": "float32",
"V296": "float32",
"V297": "float32",
"V298": "float32",
"V299": "float32",
"V300": "float32",
"V301": "float32",
"V302": "float32",
"V303": "float32",
"V304": "float32",
"V305": "float32",
"V306": "float32",
"V307": "float32",
"V308": "float32",
"V309": "float32",
"V310": "float32",
"V311": "float32",
"V312": "float32",
"V313": "float32",
"V314": "float32",
"V315": "float32",
"V316": "float32",
"V317": "float32",
"V318": "float32",
"V319": "float32",
"V320": "float32",
"V321": "float32",
"V322": "float32",
"V323": "float32",
"V324": "float32",
"V325": "float32",
"V326": "float32",
"V327": "float32",
"V328": "float32",
"V329": "float32",
"V330": "float32",
"V331": "float32",
"V332": "float32",
"V333": "float32",
"V334": "float32",
"V335": "float32",
"V336": "float32",
"V337": "float32",
"V338": "float32",
"V339": "float32",
"id_01": "float32",
"id_02": "float32",
"id_03": "float32",
"id_04": "float32",
"id_05": "float32",
"id_06": "float32",
"id_07": "float32",
"id_08": "float32",
"id_09": "float32",
"id_10": "float32",
"id_11": "float32",
"id_12": "object",
"id_13": "float32",
"id_14": "float32",
"id_15": "object",
"id_16": "object",
"id_17": "float32",
"id_18": "float32",
"id_19": "float32",
"id_20": "float32",
"id_21": "float32",
"id_22": "float32",
"id_23": "object",
"id_24": "float32",
"id_25": "float32",
"id_26": "float32",
"id_27": "object",
"id_28": "object",
"id_29": "object",
"id_30": "object",
"id_31": "object",
"id_32": "float32",
"id_33": "object",
"id_34": "object",
"id_35": "object",
"id_36": "object",
"id_37": "object",
"id_38": "object",
"DeviceType": "object",
"DeviceInfo": "object",
"is_fraud":			 "int8"
}
emails = {'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes
'.format(name, round(( time.time() - t0)/60,2)))
 def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
def fraud_preprocessing(debug = None):
print("Starting Pre-Processing.. ")
 with timer("Load Tables"):
train_transaction = pd.read_csv('.. /input/train_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 test_transaction = pd.read_csv('.. /input/test_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv',
index_col='TransactionID',
nrows= debug)
 with timer("Merge Tables"):
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
y = train[target_var].copy()
del train_transaction, train_identity, test_transaction, test_identity
traindex = train.index
testdex = test.index
with timer("Train/Test Split Feature Engineering"):
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 train = id_split(train)
 test = id_split(test)
 df = pd.concat([train.drop(target_var,axis=1),test],axis = 0)
 del train, test
with timer("Whole Feature Engineering"):
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 df = df.assign(
 TransactionAmt_decimal =(( df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000 ).astype(int),
card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)) ,
Transaction_day_of_week = np.floor(( df['TransactionDT'] /(3600 * 24)- 1)% 7),
Transaction_hour = np.floor(df['TransactionDT'] / 3600)% 24,
TransactionDT = df['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x))), 
)
 df = df.assign(
 year = df['TransactionDT'].dt.year,
month = df['TransactionDT'].dt.month,
dow = df['TransactionDT'].dt.dayofweek,
quarter = df['TransactionDT'].dt.quarter,
hour = df['TransactionDT'].dt.hour,
day = df['TransactionDT'].dt.day,
all_group_nan_sum = df.isnull().sum(axis=1)/ df.shape[1],
all_group_0_count =(df == 0 ).astype(int ).sum(axis=1)/(df.shape[1] - df.isnull().sum(axis=1)) 
)
 prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']
for i, p in enumerate(prefix):
column_set = [x for x in df.columns.tolist() if x.startswith(prefix[i])]
df[p + "group_nan_sum"] = df[column_set].isnull().sum(axis=1)/ df[column_set].shape[1]
numeric_cols = [x for x in column_set if df[x].dtype != object]
if numeric_cols:
df[p + "group_sum"] = df[column_set].sum(axis=1)
 df[p + "group_mean"] = df[column_set].mean(axis=1)
 df[p + "group_0_count"] =(df[column_set] == 0 ).astype(int ).sum(axis=1)/(df[column_set].shape[1] - df[p + "group_nan_sum"])
 with timer("Label Encode"):
categorical_cols = []
for f in df.columns:
if df[f].dtype=='object':
categorical_cols += [f]
lbl = preprocessing.LabelEncoder()
df[f] = lbl.fit_transform(df[f].astype(str))
print("Total Shape: {} Rows, {} Columns".format(*df.shape))
return df, y, traindex, testdex, categorical_cols, sample_submission<split>
from utils.datasets import *
from utils.utils import *
max_iter = 35
initial_iter = 5
DEBUG = None
df, y, traindex, testdex, cat_cols, sample_submission = fraud_preprocessing(debug = DEBUG )<feature_engineering>
def detect1Image_aug(im0, imgsz, model, device, conf_thres, iou_thres):
img = letterbox(im0, new_shape=imgsz)[0]
img = img[:, :, ::-1].transpose(2, 0, 1)
 img = np.ascontiguousarray(img)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=True)[0]
pred = non_max_suppression(pred, conf_thres, iou_thres)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )
df['yrmth'] = df.year.astype(str)+ df.month.map("{:02}".format)
 df['Fraud'] = np.nan
df.loc[traindex,'Fraud'] = y[traindex]
df['traintest'] = 'Test'
df.loc[df.Fraud.notnull() ,'traintest'] = 'Train'<count_values>
def clip_coords2(boxes, img_shape):
boxes[:, 0].clamp_(0, img_shape[1])
 boxes[:, 1].clamp_(0, img_shape[0])
 boxes[:, 2].clamp_(0, img_shape[1])
 boxes[:, 3].clamp_(0, img_shape[0])
 def scale_coords2(coords,factorx,factory, img0_shape):
coords[:, 0::2] *= factorx
coords[:, 1::2] *= factory
clip_coords2(coords, img0_shape)
 return coords
print("Are there redundant Transaction IDs?")
 print(df.index.value_counts().value_counts() )<split>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
X = df.loc[traindex,:]
feature_subset = X.columns.tolist()
test = df.loc[testdex,:]
split_size = 0.35
n_estimators = 10000
metric = 'auc'
ESR = 150
feature_subset = [x for x in X.columns.tolist() if x not in ['TransactionDT','Fraud', 'traintest', 'yrmth']]
X_train, X_valid, y_train, y_valid = train_test_split(
 X.loc[:,feature_subset], y, test_size=split_size,
random_state=seed, shuffle=False)
 del df ; gc.collect() ;
print("None Fraud: {}%, Fraud: {}%".format(*y.value_counts(normalize=True)))
 print("Randomness Score AUC: {}".format(
 metrics.roc_auc_score(y,np.array([y.value_counts(normalize=True)[0]]*y.shape[0]))))<define_variables>
!pip install /kaggle/input/orkatzfdata/yacs-0.1.7-py3-none-any.whl
!mkdir fvcore
!cp -R '/kaggle/input/orkatzfdata/fvcore-0.1.dev200407/fvcore-0.1.dev200407/'./fvcore
!pip install fvcore/fvcore-0.1.dev200407/.
 !mkdir detectron2-ResNeSt
!cp -R /kaggle/input/orkatzfdata/detectron2-ResNeSt/*./detectron2-ResNeSt/
!pip install detectron2-ResNeSt/ .
bds = [ {'name': 'min_data_in_leaf', 'type': 'continuous', 'domain':(2, 100)},
{'name': 'num_leaves', 'type': 'continuous', 'domain':(20, 1000)},
{'name': 'subsample_for_bin', 'type': 'continuous', 'domain':(1000, 5000)},
{'name': 'min_sum_hessian_in_leaf', 'type': 'continuous', 'domain':(0, 15)},
{'name': 'reg_alpha', 'type': 'continuous', 'domain':(0, 3)},
{'name': 'reg_lambda', 'type': 'continuous', 'domain':(0, 3)},
{'name': 'bagging_fraction', 'type': 'continuous', 'domain': (.01, 1)},
{'name': 'feature_fraction', 'type': 'continuous', 'domain': (.01, 1)}
]
sb_cat_cols = [x for x in cat_cols if x in feature_subset]
lgtrain = lgb.Dataset(X_train, y_train, categorical_feature = sb_cat_cols, free_raw_data=False)
 lgvalid = lgb.Dataset(X_valid, y_valid, categorical_feature = sb_cat_cols, free_raw_data=False)
 def lgb_score(para):
parameters = para[0]
modelstart= time.time()
params = {
'objective': 'binary',
'metric': metric,
'learning_rate': 0.05,
'device': 'gpu',
'tree_learner': 'serial',
'boost_from_average': 'true',
'num_boost_round': n_estimators,
'boosting_type': 'gbdt',
'max_depth': -1,
'num_leaves': int(parameters[1]),
'subsample_for_bin': int(parameters[2]),
'min_data_in_leaf': int(parameters[0]),
'min_sum_hessian_in_leaf': parameters[3],
'reg_alpha': parameters[4],
'reg_lambda': parameters[5],
'bagging_fraction': parameters[6],
'bagging_freq': 1,
'feature_fraction': parameters[7],
'is_unbalance': False,
'nthread': -1,
'verbose': -1,
'seed': seed
}
experiment_lgb = lgb.train(
 params = params,
train_set = lgtrain,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
verbose_eval= 0,
early_stopping_rounds= ESR 
)
 runtime =(time.time() - modelstart)/60
val_pred = experiment_lgb.predict(X_valid)
 score = experiment_lgb.best_score['valid'][metric]
loss = metrics.log_loss(y_valid, val_pred)
 params['num_boost_round'] = experiment_lgb.best_iteration
gpyopt_output.append(
 [
loss,
experiment_lgb.best_score['train'][metric],
score,
experiment_lgb.best_iteration,
params,
runtime
] 
)
 return score<choose_model_class>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
def load_net7(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net = torch.nn.DataParallel(net)
 checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
gpyopt_output = []
optimizer = BayesianOptimization(f=lgb_score,
domain=bds,
model_type='GP',
optimize_restarts = 1,
initial_design_numdata = initial_iter,
acquisition_type ='EI',
acquisition_jitter = 0.1,
exact_feval=True,
maximize=True)
 with timer("Bayesian Optimisation - {} Iterations".format(max_iter + initial_iter)) :
optimizer.run_optimization(max_iter=max_iter )<save_to_csv>
models2 = [load_net7('.. /input/tempb7/best-checkpoint-015epoch.bin'),
load_net7('.. /input/tempb7/best-checkpoint-020epoch.bin'),
load_net7('.. /input/tempb7/best-checkpoint-022epoch.bin'),]
models = [
load_net('.. /input/effdetbestpth/best-fold0-augmix.pth'),
load_net('.. /input/effdetbestpth/best-fold3.pth'),
load_net('.. /input/effdetbestpth/best-fold4.pth'),
load_net('.. /input/effdetbestpth/best-fold1-augmix.pth'),
load_net('.. /input/effdetbestpth/best-fold2-augmix.pth'),
load_net('.. /input/effdetbestpth/fold0-best.bin')
 ]
results = pd.DataFrame(gpyopt_output,
columns = ['logloss','train_auc','valid_auc',
'boosting_rounds','parameters', 'runtime'] 
)
 results.to_csv("gpyopt_iterations_output.csv")
 best_params = results['parameters'].iloc[np.argmax(results.valid_auc)]
results = results.reset_index().rename(columns = {'index':'iteration'})
 results['iteration'] = results['iteration'] + 1
optimizer.plot_convergence()
print("Best AUC: {}".format(optimizer.fx_opt))
print("Best Parameters")
 pprint.pprint(best_params)
 results = pd.concat([results.drop('parameters',axis=1 ).reset_index(drop=True),
json_normalize(results['parameters'] ).reset_index(drop=True)
 ], axis = 1)
 results['TPM'] = results['boosting_rounds'] / results['runtime']
results['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']<train_model>
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_cascade_rcnn_ResNeSt_101_FPN_syncbn_range-scale_1x.yaml"))
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.MODEL.WEIGHTS = os.path.join('/kaggle/input/best-inrae-1/', "model_final.pth")
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.47
cfg.DATASETS.TEST =("m5_val",)
 predictor1 = DefaultPredictor(cfg )
bds<feature_engineering>
models3 =[predictor1]
results['TPM'] = results['boosting_rounds'] / results['runtime']
results['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']
results.head()<prepare_x_and_y>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None,transforms2=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
self.transforms2 = transforms2
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image0 = sample['image']
if self.transforms2:
sample = {'image': image}
sample = self.transforms2(**sample)
 image1 = sample['image']
return image0,image1, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 def get_valid_transforms2() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() ,transforms2=get_valid_transforms2() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
allmodelstart= time.time()
EPOCHS = 5
del best_params['num_boost_round']
best_params['learning_rate'] = 0.01
kf = KFold(n_splits = EPOCHS, shuffle = False)
 y_preds = np.zeros(sample_submission.shape[0])
 y_preds_fold = np.zeros([sample_submission.shape[0],EPOCHS])
 y_oof = np.zeros(X.shape[0])
 f,ax = plt.subplots(1,3,figsize = [15,6])
 sb_cat_cols = [x for x in cat_cols if x in feature_subset]
all_feature_importance_df = pd.DataFrame()
mean_fpr = np.linspace(0,1,100)
 cms, tprs, aucs, y_real, y_proba,recalls, roc_aucs,f1_scores, accuracies, precisions = [],[],[],[],[],[],[],[],[],[]
for i,(tr_idx, val_idx)in enumerate(kf.split(X, y)) :
i += 1
best_params[seed] = i
modelstart= time.time()
evals_result = {}
lgtrain = lgb.Dataset(X.iloc[tr_idx, :][feature_subset], y.iloc[tr_idx], categorical_feature = sb_cat_cols)
 lgvalid = lgb.Dataset(X.iloc[val_idx, :][feature_subset], y.iloc[val_idx], categorical_feature = sb_cat_cols)
 clf = lgb.train(
 best_params,
lgtrain,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
verbose_eval=300,
num_boost_round = n_estimators,
early_stopping_rounds=ESR,
evals_result=evals_result, 
)
 y_oof[val_idx] = clf.predict(X.iloc[val_idx, :][feature_subset])
 y_preds_fold[:,i-1] = clf.predict(test.loc[:,feature_subset])
 y_preds += y_preds_fold[:,i-1] / EPOCHS
evals_result['train_{}'.format(i)] = evals_result.pop('train')
 evals_result['valid_{}'.format(i)] = evals_result.pop('valid')
 lgb.plot_metric(evals_result, metric=metric, ax = ax[0])
 ax[0].set_title("GPU LGBM Metric Convergence over {} Folds".format(EPOCHS))
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feature_subset
fold_importance_df["importance"] = clf.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 print("
Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
roc_aucs.append(roc_auc_score(y.iloc[val_idx].values,y_oof[val_idx]))
accuracies.append(accuracy_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
recalls.append(recall_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
precisions.append(precision_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
f1_scores.append(f1_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
fpr, tpr, t = roc_curve(y.iloc[val_idx].values,y_oof[val_idx])
 tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
 aucs.append(roc_auc)
 ax[1].plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(i,roc_auc))
precision, recall, _ = precision_recall_curve(y.iloc[val_idx].values,y_oof[val_idx])
 y_real.append(y.iloc[val_idx].values)
 y_proba.append(y_oof[val_idx])
 ax[2].plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' %(i))
cms.append(confusion_matrix(y.iloc[val_idx].values,y_oof[val_idx].round()))
ax[1].plot([0,1],[0,1], linestyle = '--', lw = 2, color = 'grey')
 mean_tpr = np.mean(tprs, axis=0)
 mean_auc = auc(mean_fpr, mean_tpr)
 ax[1].plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC(AUC = %0.4f)' %(np.mean(roc_aucs)) , lw=2, alpha=1)
 ax[1].set_xlabel('False Positive Rate')
 ax[1].set_ylabel('True Positive Rate')
 ax[1].set_title('LGB ROC curve by folds')
 ax[1].legend(loc="lower right")
 ax[2].plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')
 y_real = np.concatenate(y_real)
 y_proba = np.concatenate(y_proba)
 precision, recall, _ = precision_recall_curve(y_real, y_proba)
 ax[2].plot(recall, precision, color='blue',
label=r'Mean P|R')
 ax[2].set_xlabel('Recall')
 ax[2].set_ylabel('Precision')
 ax[2].set_title('P|R curve by folds')
 ax[2].legend(loc="lower left")
 plt.tight_layout(pad=0)
 plt.savefig('model_eval.png')
 plt.show()
print(
 'CV roc score : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)) ,
'
CV accuracy score : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)) ,
'
CV recall score : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)) ,
'
CV precision score : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)) ,
'
CV f1 score : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores))
)<save_to_csv>
def make_predictions(
 images, images1,image_ids,
score_threshold=0.25, 
):
images = images.cuda().float()
images1 = images1.cuda().float()
image_id = image_ids
rh,rw,_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg' ).shape
Hscale512 = rh/512
Wscale512 = rw/512
Hscale1024 = rh/1024
Wscale1024 = rw/1024
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes[:, 0::2] *= Wscale512
boxes[:, 1::2] *= Hscale512
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 for fold_number, net in enumerate(models2):
with torch.no_grad() :
det = net(images1, torch.tensor([1]*images1.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes[:, 0::2] *= Wscale1024
boxes[:, 1::2] *= Hscale1024
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 for fold_number, net in enumerate(models3):
image_id = image_ids
result =[]
with torch.no_grad() :
im = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 outputs = predictor1(im)
 out = outputs["instances"].to("cpu")
 scores = out.get_fields() ['scores'].numpy()
boxes = out.get_fields() ['pred_boxes'].tensor.numpy().astype(int)
 labels= out.get_fields() ['scores'].numpy()
result.append({
'boxes': boxes,
'scores': scores,
})
 predictions.append(result)
 predictions.append(result)
 return predictions
assert X[feature_subset].shape[1] == test[feature_subset].shape[1]
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('{}_feats_{}fold_lgbm_gpu.csv'.format(len(feature_subset),EPOCHS))<train_model>
def clip_coords3(boxes, img_shape):
boxes[:, 0].clamp_(0, img_shape[1])
 boxes[:, 1].clamp_(0, img_shape[0])
 boxes[:, 2].clamp_(0, img_shape[1])
 boxes[:, 3].clamp_(0, img_shape[0])
 return boxes
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))<set_options>
def run_wbf2(predictions, image_index, image_size=1024, iou_thr=0.34, skip_box_thr=0.33, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
RANDOM_SEED = 314
rcParams['figure.figsize'] = 14, 8.7
LABELS = ["Normal","Fraud"]
sns.set()<load_from_csv>
def detect() :
transforms = get_valid_transforms()
transforms2 = get_valid_transforms2()
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
weights800 = '.. /input/yolo800/best_yolov5x_fold0_800.pt'
if not os.path.exists(weights):
weights = '.. /input/yolov5pth/weightsbest_yolov5x_fold3.pt'
imgsz = 1024
conf_thres = 0.34
iou_thres = 0.45
is_TTA = True
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
model800 = torch.load(weights800, map_location=device)['model'].float()
model800.to(device ).eval()
dataset = LoadImages(source, img_size=imgsz)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for name in tqdm(imagenames):
image_id = name.split('.')[0]
im01 = cv2.imread('%s/%s.jpg'%(source,image_id))
image = cv2.cvtColor(im01, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
sample = {'image': image}
sample = transforms(**sample)
 image0 = sample['image']
sample = {'image': image}
sample = transforms2(**sample)
 image1 = sample['image']
if image1.ndimension() == 3:
image1 = image1.unsqueeze(0)
 image0 = image0.unsqueeze(0)
 assert im01 is not None, 'Image Not Found '
im_w, im_h = im01.shape[:2]
if is_TTA:
enboxes = []
enscores = []
for i in range(4):
im0 = TTAImage(im01, i)
 boxes, scores = detect1Image(im0, imgsz, model, device, conf_thres, iou_thres)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, im_w, im_h)
 if 1:
enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 enboxes.append(boxes)
 enscores.append(scores)
 enboxes.append(boxes)
 enscores.append(scores)
 predictions = make_predictions(image0,image1,image_id)
 boxes, scores, labels = run_wbf2(predictions, image_index=0)
 enboxes.append(boxes)
 enscores.append(scores)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w, iou_thr=0.48, skip_box_thr=0.575)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=im_w)
 else:
boxes, scores = detect1Image(im01, imgsz, model, device, conf_thres, iou_thres)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= 0.05].astype(np.int32)
 scores = scores[scores >=float(0.05)]
if count<10:
for box, score in zip(boxes,scores):
cv2.rectangle(im0, 
(box[0], box[1]), 
(box[2]+box[0], box[3]+box[1]), 
(220, 0, 0), 2)
 cv2.putText(im0, '%.2f'%(score),(box[0]+np.random.randint(20), box[1]), cv2.FONT_HERSHEY_SIMPLEX ,
0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(im0)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results
<feature_engineering><EOS> 
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()