Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<define_variables>
! pip install.. /input/mmdetectionv260/addict-2.4.0-py3-none-any.whl
! pip install.. /input/mmdetectionv260/mmcv_full-latesttorch1.6.0cu102-cp37-cp37m-manylinux1_x86_64.whl
! pip install.. /input/mmdetectionv260/mmpycocotools-12.0.3-cp37-cp37m-linux_x86_64.whl
! pip install.. /input/mmdetection-package/mmdet-2.7.0-py3-none-any.whl
for c in ['signal_batch_mean','signal_batch_median',
'mean_abs_chg_batch','abs_max_batch','abs_min_batch', 'abs_median_batch',
'moving_average_batch_1000_mean',
'range_batch','abs_avg_batch',
'signal_batch_5k_mean', 'signal_batch_5k_median', 'signal_batch_5k_mad',
'moving_average_batch_5k_1000_mean',
'mean_abs_chg_batch_5k','abs_max_batch_5k', 'abs_min_batch_5k', 'abs_median_batch_5k',
'range_batch_5k','abs_avg_batch_5k',
'signal_shift+1', 'signal_shift+2', 'signal_shift-1', 'signal_shift-2',
'signal_shift+1_5k', 'signal_shift+2_5k', 'signal_shift-1_5k', 'signal_shift-2_5k',
'abs_max_signal_shift+1_5k', 'abs_max_signal_shift-1_5k'
]:
all_data[f'{c}_msignal'] = all_data[c] - all_data['signal']<define_variables>
! pip install.. /input/mmdetection-package/torch-1.6.0-cp37-cp37m-linux_x86_64.whl
FEATURES = [f for f in all_data.columns if f not in ['open_channels','index','time','train','batch',
'signal_batch_max', 'signal_batch_mad', 'maxtomin_batch'
'batch_index','batch_slices','batch_slices2', 'median_abs_chg_batch_5k',
'abs_mean_batch', 'abs_median_batch', 'abs_avg_batch', 'signal_batch_median',
'abs_max_batch', 'abs_max_batch_5k', 'abs_median_batch_5k',
'moving_average_batch_1000_mean', 'moving_average_batch_5k_1000_mean']]
print('.... : FEATURE LIST :.... ')
 print([f for f in FEATURES])
 print(f"Features: {len(FEATURES)}" )<prepare_x_and_y>
TARGET = 'open_channels'
all_data['train'] = all_data['train'].astype('bool')
 train_df = all_data.query('train' ).copy()
test_df = all_data.query('not train' ).copy()
train_df['open_channels'] = train_df['open_channels'].astype(int)
 X = train_df[FEATURES]
X_test = test_df[FEATURES]
y = train_df[TARGET].values
sub = test_df[['time']].copy()
groups = train_df['batch']
oof_df = train_df[['signal','open_channels']].copy()<drop_column>
! cp -r.. /input/mmdetection-wheat-models/attention_stage1/faster_rcnn_r50_fpn_attention_0010_dcn_albu_1x4_1x_bWheat_kaggle.py./config.py
! cp -r.. /input/mmdetection-wheat-models/attention_stage1/epoch_12.pth./model.pth
del all_data
del train_df
del test_df
gc.collect()<init_hyperparams>
CONFIG_FILE = './config.py'
CHECKPOINT_PATH = './model.pth'
TEST_IMG_DIR = '.. /input/global-wheat-detection/test'
TOTAL_FOLDS = 7
MODEL_TYPE = 'LGBM'
SHUFFLE = True
NUM_BOOST_ROUND = 3_500
EARLY_STOPPING_ROUNDS = 50
VERBOSE_EVAL = 500
RANDOM_SEED = 99943
params = {'learning_rate': 0.05,
'max_depth': -1,
'num_leaves': 2**8+1,
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'bagging_freq': 0,
'n_jobs': 8,
'seed': RANDOM_SEED,
'metric': 'rmse',
'objective' : 'regression',
'num_class': 1
}
kfold = KFold(n_splits=TOTAL_FOLDS, shuffle=SHUFFLE, random_state=RANDOM_SEED)
 feature_importance_df = pd.DataFrame()
fold_ = 1
for tr_idx, val_idx in kfold.split(X, y, groups=groups):
print(f'====== Fold {fold_:0.0f} of {TOTAL_FOLDS} ======')
 X_tr, X_val = X.iloc[tr_idx], X.iloc[val_idx]
y_tr, y_val = y[tr_idx], y[val_idx]
train_set = lgb.Dataset(X_tr, y_tr)
 val_set = lgb.Dataset(X_val, y_val)
 model = lgb.train(params,
train_set,
num_boost_round = NUM_BOOST_ROUND,
early_stopping_rounds = EARLY_STOPPING_ROUNDS,
valid_sets = [train_set, val_set],
verbose_eval = VERBOSE_EVAL)
 preds = model.predict(X_val, num_iteration=model.best_iteration)
 preds = np.round(np.clip(preds, 0, 10)).astype(int)
 test_preds = model.predict(X_test, num_iteration=model.best_iteration)
 test_preds = np.round(np.clip(test_preds, 0, 10)).astype(int)
 oof_df.loc[oof_df.iloc[val_idx].index, 'oof'] = preds
sub[f'open_channels_fold{fold_}'] = test_preds
f1 = f1_score(oof_df.loc[oof_df.iloc[val_idx].index]['open_channels'],
oof_df.loc[oof_df.iloc[val_idx].index]['oof'],
average = 'macro')
 rmse = np.sqrt(mean_squared_error(oof_df.loc[oof_df.index.isin(val_idx)]['open_channels'],
oof_df.loc[oof_df.index.isin(val_idx)]['oof']))
fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = FEATURES
fold_importance_df["importance"] = model.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print(f'Fold {fold_} - validation f1: {f1:0.5f}')
 print(f'Fold {fold_} - validation rmse: {rmse:0.5f}')
 fold_ += 1
oof_f1 = f1_score(oof_df['open_channels'],
oof_df['oof'],
average = 'macro')
 oof_rmse = np.sqrt(mean_squared_error(oof_df['open_channels'],
oof_df['oof']))<save_to_csv>
import os
import json
import pandas as pd
import numpy as np
import cv2
from tqdm import tqdm
import torch
import mmcv
from mmdet.apis import init_detector, inference_detector
s_cols = [s for s in sub.columns if 'open_channels' in s]
sub['open_channels'] = sub[s_cols].median(axis=1 ).astype(int)
 sub[['time','open_channels']].to_csv('./submission.csv',
index=False,
float_format='%0.4f' )<define_variables>
TEST_ANN_FILE = './annotation_test.json'
anns = []
for img_name in tqdm(os.listdir(TEST_IMG_DIR)) :
if not img_name.endswith('.jpg'):
continue
anns.append(dict(filename=img_name, boxes=[]))
%matplotlib inline
path_data = "/kaggle/input/ashrae-energy-prediction/"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"<load_from_csv>
with open('./result.bbox.json', 'r')as f1, open(TEST_ANN_FILE, 'r')as f2:
result_info = json.load(f1)
 annotations_info = json.load(f2)
 for i, ann in tqdm(enumerate(result_info)) :
if ann['score'] < 0.5:
continue
annotation = ann
annotation['id'] = i
annotation['area'] = ann['bbox'][2] * ann['bbox'][3]
annotation['iscrowd'] = 0
annotations_info['annotations'].append(annotation)
 mmcv.dump(annotations_info, './annotation_new.json' )
train_df = pd.read_csv(path_train)
 building_df = pd.read_csv(path_building)
 le = LabelEncoder()
building_df.primary_use = le.fit_transform(building_df.primary_use)
 weather_train_df = pd.read_csv(path_weather_train )<data_type_conversions>
train_config = './train_config.py'
cfg = mmcv.Config.fromfile(CONFIG_FILE)
 cfg.data.samples_per_gpu = 8
cfg.data.workers_per_gpu = 4
cfg.data.train.ann_file = './annotation_new.json'
cfg.data.train.img_prefix = TEST_IMG_DIR
cfg.data.train.pipeline = [
dict(type='LoadImageFromFile'),
dict(type='LoadAnnotations', with_bbox=True),
dict(
 type='Albu',
transforms=cfg.albu_train_transforms,
bbox_params=dict(
 type='BboxParams',
format='pascal_voc',
label_fields=['gt_labels'],
min_visibility=0.3,
filter_lost_elements=True),
keymap=dict(
 img='image',
gt_bboxes='bboxes'),
update_pad_shape=False,
skip_img_without_anno=True),
dict(type='Resize', img_scale=(1024, 1024), keep_ratio=True),
dict(type='RandomFlip', flip_ratio=0.5),
dict(type='Normalize', **cfg.img_norm_cfg),
dict(type='Pad', size_divisor=32),
dict(type='DefaultFormatBundle'),
dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),
]
cfg.optimizer = dict(type='SGD', lr=0.005, momentum=0.9, weight_decay=0.0001)
 cfg.lr_config = dict(
 policy='fixed',
warmup=None 
)
 cfg.total_epochs = 2
cfg.model.pretrained = None
cfg.load_from = CHECKPOINT_PATH
cfg.log_config = dict(
 interval=1,
hooks=[
dict(type='TextLoggerHook')
 ])
 with open(train_config, 'w')as f:
f.write(cfg.pretty_text )
def preparing_data(X, bld_data, wth_data, test=False):
X = X.merge(bld_data, on = "building_id", how = "left")
 X = X.merge(wth_data, on = ["site_id", "timestamp"], how = "left")
 X["timestamp"] = pd.to_datetime(X["timestamp"], format="%Y-%m-%d %H:%M:%S")
 X["square_feet"] = np.log1p(X["square_feet"])
 if not test:
X.sort_values("timestamp", inplace = True)
 X.reset_index(drop=True, inplace = True)
 X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04","2016-09-05", "2016-10-10",
"2016-11-11", "2016-11-24", "2016-12-26", "2017-01-01", "2017-01-16", "2017-02-20", "2017-05-29",
"2017-07-04", "2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25", "2018-01-01",
"2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04", "2018-09-03", "2018-10-08", "2018-11-12",
"2018-11-22", "2018-12-25", "2019-01-01"]
X["isholiday"] =(X.timestamp.dt.date.astype("str" ).isin(holidays)).astype(int)
 X.drop("timestamp", axis=1, inplace=True)
 if test:
row_ids = X.row_id
X.drop("row_id", axis=1, inplace=True)
 return X, row_ids
else:
y = np.log1p(X.meter_reading)
 X.drop("meter_reading", axis=1, inplace=True)
 return X, y<prepare_x_and_y>
! python./mmdetection/tools/train.py {train_config} --no-validate --work-dir./pseudo
X_full, y_full = preparing_data(train_df, building_df, weather_train_df)
 del train_df, weather_train_df
gc.collect()<create_dataframe>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
categorical_features = ["building_id", "site_id", "meter", "primary_use", "hour", "weekday", "isholiday"]
X_half1 = X_full.iloc[:int(len(X_full)/2)]
X_half2 = X_full[int(len(X_full)/2):]
y_half1 = y_full[:int(len(X_full)/2)]
y_half2 = y_full[int(len(X_full)/2):]
lgb_train1_data = lgb.Dataset(X_half1, label=y_half1, categorical_feature=categorical_features, free_raw_data=False)
 lgb_train2_data = lgb.Dataset(X_half2, label=y_half2, categorical_feature=categorical_features, free_raw_data=False)
 parameters = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 100,
"learning_rate": 0.05,
"feature_fraction": 0.80,
"reg_lambda": 2,
"metric": "rmse",
"n_jobs": 4
}
print("start building model1_1 : ")
 model1_1 = lgb.train(parameters, train_set=lgb_train1_data, num_boost_round=1000, valid_sets = [lgb_train1_data, lgb_train2_data],
verbose_eval=200, early_stopping_rounds=200)
 print("start building model1_2 : ")
 model1_2 = lgb.train(parameters, train_set=lgb_train2_data, num_boost_round=1000, valid_sets=[lgb_train2_data, lgb_train1_data],
verbose_eval=200, early_stopping_rounds=200 )<drop_column>
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
del X_half1, X_half2, y_half1, y_half2, lgb_train1_data, lgb_train2_data, categorical_features
gc.collect()<load_from_csv>
config = mmcv.Config.fromfile(CONFIG_FILE)
 config.model.pretrained = None
config.data.test.pipeline[1]['img_scale'] = [(1280, 1280),(1408, 1408)]
model = init_detector(config, './pseudo/epoch_1.pth', device=device)
 model.eval()
test_full = pd.read_csv(path_test)
 weather_test = pd.read_csv(path_weather_test)
 test_full = reduce_mem_usage(test_full)
 weather_test = reduce_mem_usage(weather_test)
 X_test, row_ids = preparing_data(test_full, building_df, weather_test, test=True )<drop_column>
results = []
score_threshold = 0.3
with torch.no_grad() :
for img_name in tqdm(os.listdir(TEST_IMG_DIR)) :
img_pth = os.path.join(TEST_IMG_DIR, img_name)
 image = mmcv.imread(img_pth)
 result = inference_detector(model, image)
 boxes = result[0][:, :4]
scores = result[0][:, 4]
if len(boxes)> 0:
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= score_threshold]
scores = scores[scores >= score_threshold]
result = {
'image_id': img_name[:-4],
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<predict_on_test><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()