Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<drop_column>
sys.path.insert(0, "/kaggle/input/weightedboxesfusion")
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<drop_column>
DATA_DIR = "/kaggle/input/global-wheat-detection"
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
rm_cols = [
'TransactionID',
'uid', 'uid2', 'uid3','uid4', 'uid5',
'bank_type',
'DT', 'DT_M', 'DT_W', 'DT_D',
]
train = train.drop(rm_cols, axis=1)
 test = test.drop(rm_cols, axis=1 )<prepare_x_and_y>
test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
test_df.shape
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1 )<set_options>
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
del train, test
gc.collect()<init_hyperparams>
def fasterrcnn_resnet50_fpn(path,pretrained_backbone=False):
backbone = resnet_fpn_backbone('resnet50', pretrained_backbone)
 model = FasterRCNN(backbone, 2)
 model.load_state_dict(torch.load(path))
model.to(DEVICE)
 model.eval()
return model
params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.85,
'subsample_freq':1,
'subsample':0.85,
'max_bin':255,
'verbose':-1,
'seed': 47,
'n_estimators':10000,
'early_stopping_rounds':500,
'reg_alpha':0.3,
'reg_lamdba':0.243
}<split>
def get_model_101(path,pretrained=False):
backbone = resnet_fpn_backbone('resnet101', pretrained=pretrained)
 model = FasterRCNN(backbone, num_classes=2)
 in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)
 model.load_state_dict(torch.load(path))
model.to(DEVICE)
 model.eval()
return model
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()<compute_test_metric>
model1 =get_model_101(".. /input/faster-cnn-final-model/fastercnn_resnet_101_145.pth")
 model2= get_model_101(".. /input/fastercnn-155/fastercnn_101_155.pth")
 model3= get_model_101(".. /input/faster-cnn-101/fastercnn_resnet_101_155_WightDecay.pth")
 faster_CNN_50_90_2=fasterrcnn_resnet50_fpn(".. /input/isfix-model/fastercnn_50_fix.pth")
 models=[model1,model3,model2,faster_CNN_50_90_2]
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<save_to_csv>
def make_ensemble_predictions(images):
images = list(image.to(DEVICE)for image in images)
 result = []
for model in models:
with torch.no_grad() :
outputs = model(images)
 result.append(outputs)
 del model
gc.collect()
torch.cuda.empty_cache()
return result
def run_wbf_ensemble(predictions, image_index, image_size=1024, iou_thr=0.34, skip_box_thr=0.43, weights=None):
boxes = [prediction[image_index]['boxes'].data.cpu().numpy() /(image_size-1)for prediction in predictions]
scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0])for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
sub['isFraud'] = y_preds
sub.to_csv("submission.csv", index=False )<save_to_csv>
def get_test_transforms() :
return A.Compose([
ToTensorV2(p=1.0)
 ], p=1.0 )
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature')
 plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits))<import_modules>
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=1,
drop_last=False,
collate_fn=collate_fn
)
import pandas as pd
import scipy as sp
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix, roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_score, KFold
from xgboost import XGBClassifier
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
import lightgbm as lgb
import os
import gc<load_from_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv')
 train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv')
 sub = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )<count_values>
class BaseWheatTTA:

 image_size = 1024
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
train.isFraud.value_counts()<randomize_order>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
not_fraud=train[train.isFraud==0]
fraud=train[train.isFraud==1]
not_fraud_downsampled = resample(not_fraud,
replace = False,
n_samples = 400000,
random_state = 27)
 downsampled = pd.concat([not_fraud_downsampled, fraud])
 downsampled.isFraud.value_counts()<drop_column>
def make_tta_predictions(images, score_threshold=0.57):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = make_ensemble_predictions(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes, scores, labels = run_wbf_ensemble(outputs, image_index=i)
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
train = downsampled.copy()
del not_fraud_downsampled, downsampled, train_identity, test_identity, train_transaction, test_transaction<drop_column>
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.6, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<feature_engineering>
results = []
list_box=[]
list_score=[]
list_image_id=[]
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
list_score.append(scores)
 list_box.append(boxes)
 list_image_id.append(image_id)
 result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"]=="samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"]=="opera 53.0",'lastest_browser']=1
df.loc[df["id_31"]=="mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"]=="google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"]=="firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"]=="edge 17.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for ios",'lastest_browser']=1
return df
train=setBrowser(train)
 test=setBrowser(test )<feature_engineering>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].cat.add_categories('unknown_device')
 df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df['had_id'] = 1
gc.collect()
return df
train=setDevice(train)
 test=setDevice(test )<data_type_conversions>
test_df.to_csv('submission.csv', index=False )