Kaggle competition: Global Wheat Detection
def lgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'regression',
'num_leaves' : 40,
'min_data_in_leaf' : 20,
'max_depth' : 4,
'learning_rate': 0.005,
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8,
"bagging_seed": random_seed,
"metric": 'rmse',
"random_state" : random_seed,
"verbosity": -1}
record = dict()
model = lgb.train(params
, lgb.Dataset(trn_x, trn_y)
 , num_boost_round = 1000000
, valid_sets = [lgb.Dataset(val_x, val_y)]
, verbose_eval = verbose
, early_stopping_rounds = 500
, callbacks = [lgb.record_evaluation(record)] 
)
 best_idx = np.argmin(np.array(record['valid_0']['rmse']))
val_pred = model.predict(val_x, num_iteration = model.best_iteration)
 test_pred = model.predict(test, num_iteration = model.best_iteration)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid_0']['rmse'][best_idx], 'importance':model.feature_importance('gain')}<compute_train_metric>
WEIGHT_FILE1 = '.. /input/yolov5multiscale/best_yolov5x_fold0.pt'
WEIGHT_FILE2 = '.. /input/yolov5multiscale/best_yolov5x_fold1.pt'
DIR_TEST = '.. /input/global-wheat-detection/test/'
def cat_model(trn_x, trn_y, val_x, val_y, test, verbose):
model = CatBoostRegressor(iterations=10000000,
learning_rate=0.005,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.7,
random_seed = random_seed,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=500 
)
 model.fit(trn_x, trn_y,
eval_set=(val_x, val_y),
use_best_model=True,
verbose=False)
 val_pred = model.predict(val_x)
 test_pred = model.predict(test)
 return {'val':val_pred, 'test':test_pred, 'error':model.get_best_score() ['validation_0']['RMSE'], 'importance':model.get_feature_importance() }<train_model>
%%time
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = False
models = []
for w in weights:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
for model in models:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, 0.4, opt.iou_thres, merge=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
class opt:
weights_folds = [WEIGHT_FILE1]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()
result_dict = dict()
val_pred = np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 final_err = 0
verbose = False
for i,(trn, val)in enumerate(fold):
print(i+1, "fold.RMSE")
 trn_x = train.loc[trn, :]
trn_y = y[trn]
val_x = train.loc[val, :]
val_y = y[val]
fold_val_pred = []
fold_test_pred = []
fold_err = []
start = datetime.now()
result = xgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("xgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = lgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("lgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = cat_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("cat model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 val_pred[val] += np.mean(np.array(fold_val_pred), axis = 0)
 test_pred += np.mean(np.array(fold_test_pred), axis = 0)/ k
final_err +=(sum(fold_err)/ len(fold_err)) / k
print("---------------------------")
 print("avg err.", "{0:.5f}".format(sum(fold_err)/ len(fold_err)))
 print("blend err.", "{0:.5f}".format(np.sqrt(np.mean(( np.mean(np.array(fold_val_pred), axis = 0)- val_y)**2))))
print('')
 print("fianl avg err.", final_err)
 print("fianl blend err.", np.sqrt(np.mean(( val_pred - y)**2)) )<save_to_csv>
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.4, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = np.expm1(test_pred)
 df_sub.to_csv("submission.csv", index=False )<install_modules>
all_path,all_score,all_bboxex = res
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
results =[]
testdf_psuedo = []
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'kaushal28'+image_id,
'source': 'kaushal28',
'x': float(box[0]),
'y': float(box[1]),
'w': float(box[2]),
'h': float(box[3]),
'x_center': float(box[0])+ float(box[2])/2,
'y_center': float(box[1])+ float(box[3])/2,
'classes': 0
}
testdf_psuedo.append(result)
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 21
seed_everything(seed)
 warnings.filterwarnings('ignore' )<define_variables>
test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'x', 'y', 'w', 'h', 'x_center','y_center','classes', 'source'])
 test_df_pseudo.head()
BATCH_SIZE = 32 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8<define_search_space>
train_df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(train_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
train_df[column] = bboxs[:,i]
train_df.drop(columns=['bbox'], inplace=True)
 train_df['x_center'] = train_df['x'] + train_df['w']/2
train_df['y_center'] = train_df['y'] + train_df['h']/2
train_df['classes'] = 0
train_df = train_df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes', 'source']]
train_df.head()
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label<train_model>
train_df = pd.concat([train_df, test_df_pseudo], ignore_index=True)
 train_df
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<choose_model_class>
%%writefile yolov5x.yaml
nc: 1
depth_multiple: 1.33
width_multiple: 1.25
anchors:
- [116,90, 156,198, 373,326]
- [30,61, 62,45, 59,119]
- [10,13, 16,30, 33,23]
backbone:
[[-1, 1, Focus, [64, 3]],
[-1, 1, Conv, [128, 3, 2]],
[-1, 3, BottleneckCSP, [128]],
[-1, 1, Conv, [256, 3, 2]],
[-1, 9, BottleneckCSP, [256]],
[-1, 1, Conv, [512, 3, 2]],
[-1, 9, BottleneckCSP, [512]],
[-1, 1, Conv, [1024, 3, 2]],
[-1, 1, SPP, [1024, [5, 9, 13]]],
]
head:
[[-1, 3, BottleneckCSP, [1024, False]],
[-1, 1, Conv, [512, 1, 1]],
[-1, 1, nn.Upsample, [None, 2, 'nearest']],
[[-1, 6], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [512, False]],
[-1, 1, Conv, [256, 1, 1]],
[-1, 1, nn.Upsample, [None, 2, 'nearest']],
[[-1, 4], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [256, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[-2, 1, Conv, [256, 3, 2]],
[[-1, 14], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [512, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[-2, 1, Conv, [512, 3, 2]],
[[-1, 10], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [1024, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[[], 1, Detect, [nc, anchors]],
]
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='relu', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary()<predict_on_test>
yolo_config = {
'epochs': 37,
'image_size': 1024,
'batch_size': 2,
'data_config': 'wheat_data_config.yaml',
'yolo_config': 'yolov5x.yaml',
'results_name': f'yolov5x_fold{FOLD}',
'weights': WEIGHT_FILE1
}
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
if len(os.listdir(DIR_TEST)) > 10:
!(python train.py --img-size {yolo_config.get('image_size')} \
--batch-size {yolo_config.get('batch_size')} \
--epochs {yolo_config.get('epochs')} \
--data {yolo_config.get('data_config')} \
--cfg {yolo_config.get('yolo_config')} \
--name {yolo_config.get('results_name')} \
--weights {yolo_config.get('weights')} )
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<install_modules>
torch.cuda.empty_cache()
!rm -rf convertor
!pip install.. /input/keras-efficientnet-whl/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/keras-efficientnet-whl/efficientnet-1.1.1-py3-none-any.whl<set_options>
class opt:
weights_folds = [WEIGHT_FILE2]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()
%matplotlib inline
print("Tensorflow version " + tf.__version__)
<define_variables>
all_path,all_score,all_bboxex = res
input_dir = ".. /input/cassava-leaf-disease-classification"
train_images_path = os.path.join(input_dir,"train_images")
 test_images_path = os.path.join(input_dir,'test_images' )<load_from_csv>
results =[]
testdf_psuedo = []
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'kaushal28'+image_id,
'source': 'kaushal28',
'x': float(box[0]),
'y': float(box[1]),
'w': float(box[2]),
'h': float(box[3]),
'x_center': float(box[0])+ float(box[2])/2,
'y_center': float(box[1])+ float(box[3])/2,
'classes': 0
}
testdf_psuedo.append(result )
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv' )<categorify>
test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'x', 'y', 'w', 'h', 'x_center','y_center','classes', 'source'])
 test_df_pseudo.head()
image_list = train['image_id'].to_list()
label_list = train['label'].to_list()<define_variables>
train_df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(train_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
train_df[column] = bboxs[:,i]
train_df.drop(columns=['bbox'], inplace=True)
 train_df['x_center'] = train_df['x'] + train_df['w']/2
train_df['y_center'] = train_df['y'] + train_df['h']/2
train_df['classes'] = 0
train_df = train_df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes', 'source']]
train_df.head()
TARGET_SIZE = 512<load_pretrained>
train_df = pd.concat([train_df, test_df_pseudo], ignore_index=True)
 train_df
model2 = keras.models.load_model('.. /input/efficb3002/results-4/efficb1.h5')
 model1 = keras.models.load_model('.. /input/effb116/results-2/efficb1.h5' )<load_from_csv>
torch.cuda.empty_cache()
yolo_config = {
'epochs': 39,
'image_size': 1024,
'batch_size': 2,
'data_config': 'wheat_data_config.yaml',
'yolo_config': 'yolov5x.yaml',
'results_name': f'yolov5x_fold{FOLD}',
'weights': WEIGHT_FILE2
}
submission_file = pd.read_csv(os.path.join('.. /input/cassava-leaf-disease-classification/sample_submission.csv'))
submission_file<predict_on_test>
if len(os.listdir(DIR_TEST)) > 10:
!(python train.py --img-size {yolo_config.get('image_size')} \
--batch-size {yolo_config.get('batch_size')} \
--epochs {yolo_config.get('epochs')} \
--data {yolo_config.get('data_config')} \
--cfg {yolo_config.get('yolo_config')} \
--name {yolo_config.get('results_name')} \
--weights {yolo_config.get('weights')} )
preds = []
for image_id in submission_file.image_id:
image = Image.open(os.path.join(f'.. /input/cassava-leaf-disease-classification/test_images/{image_id}'))
image = image.resize(( TARGET_SIZE, TARGET_SIZE))
image = np.expand_dims(image, axis = 0)
 best_alpha = 0.51
probabilities1 = model1.predict(image)
 probabilities2 = model2.predict(image)
 probabilities = best_alpha * probabilities1 +(1 - best_alpha)* probabilities2
predictions = np.argmax(probabilities, axis=-1)
 preds.append(predictions[0])
 submission_file['label'] = preds
submission_file<save_to_csv>
class opt:
weights_folds = list(glob.glob("weights/best_*")) if len(os.listdir(DIR_TEST)) > 10 else [WEIGHT_FILE1, WEIGHT_FILE2]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()
submission_file.to_csv('submission.csv', index = False )<import_modules>
all_path,all_score,all_bboxex = res
print("Tensorflow version " + tf.__version__ )<define_variables>
results =[]
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'] )
IMAGE_SIZE = 512
BATCH_SIZE = 64<define_variables>
test_df.to_csv('submission.csv', index=False)
 test_df.head()