Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<merge>
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 print("Train shape: ", train.shape)
 print("Test shape:", test.shape)
 y = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
gc.collect()
train = train.drop('isFraud', axis=1 )<drop_column>
!pip install --no-deps '.. /input/timm0130/timm-0.1.30-py3-none-any.whl' > /dev/null
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<define_variables>
%matplotlib inline
VerticalFlip, HorizontalFlip, IAASharpen,
OneOf, Compose , BboxParams, Resize, HueSaturationValue
,RandomBrightnessContrast, ToGray , Cutout , RandomSizedCrop )
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']<drop_column>
from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain , DetBenchPredict
from effdet.efficientdet import HeadNet
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('TransactionID')
 cols_to_drop.remove('TransactionDT' )<drop_column>
DIR_PATH = '.. /input/global-wheat-detection/'
dir = glob.glob(os.path.join(DIR_PATH , '*'))
dir.sort(reverse=True)
 print(dir)
 train_paths = glob.glob(os.path.join(dir[1] , '*'))
test_paths = glob.glob(os.path.join(dir[2] , '*'))
print('{} features are going to be dropped for being useless'.format(len(cols_to_drop)))
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<data_type_conversions>
df = pd.read_csv(dir[0])
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df
def addNewFeatures(data):
data['uid1'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid1'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['D9'] = np.where(data['D9'].isna() ,0,1)
 return data
train = addNewFeatures(train)
 test = addNewFeatures(test)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour_of_day'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour_of_day'] = np.floor(test['TransactionDT'] / 3600)% 24
train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<count_values>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = df[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = df[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds
train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['card2_count_full'] = train['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
test['card2_count_full'] = test['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
train['card3_count_full'] = train['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
test['card3_count_full'] = test['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
train['card4_count_full'] = train['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
test['card4_count_full'] = test['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
train['card5_count_full'] = train['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
test['card5_count_full'] = test['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
train['card6_count_full'] = train['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
test['card6_count_full'] = test['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
train['addr1_count_full'] = train['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
test['addr1_count_full'] = test['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
train['addr2_count_full'] = train['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))
test['addr2_count_full'] = test['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))<categorify>
df_folds.groupby('fold')['bbox_count'].mean()
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std' )<feature_engineering>
image_id = '8425a537b.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image without bounding box" )
train['TransactionAmt'] = np.log1p(train['TransactionAmt'])
 test['TransactionAmt'] = np.log1p(test['TransactionAmt'] )<data_type_conversions>
image_id = 'b3c96d5ad.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image with bounding box" )
dates_range = pd.date_range(start='2017-10-01', end='2019-01-01')
 us_holidays = calendar().holidays(start=dates_range.min() , end=dates_range.max())
 def setTime(df):
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(( df['DT'].dt.year-2017)*12 + df['DT'].dt.month ).astype(np.int8)
 df['DT_W'] =(( df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear ).astype(np.int8)
 df['DT_D'] =(( df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear ).astype(np.int16)
 df['DT_hour'] =(df['DT'].dt.hour ).astype(np.int8)
 df['DT_day_week'] =(df['DT'].dt.dayofweek ).astype(np.int8)
 df['DT_day_month'] =(df['DT'].dt.day ).astype(np.int8)
 df['is_december'] = df['DT'].dt.month
df['is_december'] =(df['is_december']==12 ).astype(np.int8)
 df['is_holiday'] =(df['DT'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 return df
train = setTime(train)
 test = setTime(test )<data_type_conversions>
images_after_augmentation(image, boxes, Blur(blur_limit=7 ,p=1))
i_cols = ['card1','card2','card3','card5','uid1','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train[new_col_name] = train[col].map(temp_df)
 test[new_col_name] = test[col].map(temp_df)
 train = train.replace(np.inf,999)
 test = test.replace(np.inf,999 )<feature_engineering>
images_after_augmentation(image, boxes, HorizontalFlip(p=1))
train['P_isproton'] =(train['P_emaildomain']=='protonmail.com')
 train['R_isproton'] =(train['R_emaildomain']=='protonmail.com')
 test['P_isproton'] =(test['P_emaildomain']=='protonmail.com')
 test['R_isproton'] =(test['R_emaildomain']=='protonmail.com' )<count_missing_values>
class WheatDataset(Dataset):
def __init__(self , dataframe , image_ids, transforms = None):
super().__init__()
self.image_ids = image_ids
self.dataframe = dataframe
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['labels'] = torch.stack(sample['labels'])
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[1]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.dataframe[self.dataframe['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
area =(boxes[:, 2] * boxes[:, 3])
 boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
train['nulls1'] = train.isna().sum(axis=1)
 test['nulls1'] = test.isna().sum(axis=1 )<define_variables>
def collate_fn(batch):
return tuple(zip(*batch))
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink', 'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other',
'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att', 'frontier.com': 'yahoo',
'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple',
'yahoo.co.uk': 'yahoo', 'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
bs = 2
fold_number = 0
train_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] != fold_number].index.values , transforms=transforms_train)
 valid_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] == fold_number].index.values , transforms=transforms_valid)
 train_loader = DataLoader(train_set,batch_size = bs,sampler=RandomSampler(train_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=2)
 valid_loader = DataLoader(valid_set,batch_size = bs ,sampler=SequentialSampler(valid_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=2)
 images , targets , path_images = next(iter(train_loader))
img = visualizeTarget(images[0],targets[0])
 visualize(Example_one_image_from_dataloader = img )
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<feature_engineering>
class Training:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.best_calc_loss = 10**5
self.model = model
self.device = device
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 def train_loop(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
print(f'
{timestamp}
LR: {lr}')
 t = time.time()
calc_loss = self.train_one_epoch(train_loader)
 print(f'Train.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save_model(f'{self.base_dir}/last-epoch.bin')
 t = time.time()
calc_loss = self.valid_one_epoch(validation_loader)
 print(f'Val.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if calc_loss.avg < self.best_calc_loss:
self.best_calc_loss = calc_loss.avg
self.model.eval()
self.save_model(f'{self.base_dir}/best-model-{str(self.epoch ).zfill(3)}epoch.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=calc_loss.avg)
 self.epoch += 1
def train_one_epoch(self, train_loader):
self.model.train()
calc_loss = Calculator()
t = time.time()
target_res = {}
for images, targets, image_ids in tqdm_notebook(train_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res['bbox'] = boxes
target_res['cls'] = labels
self.optimizer.zero_grad()
outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 loss.backward()
self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return calc_loss
def valid_one_epoch(self, val_loader):
self.model.eval()
calc_loss = Calculator()
t = time.time()
with torch.no_grad() :
for images, targets, image_ids in tqdm_notebook(val_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res = {}
target_res['bbox'] = boxes
target_res['cls'] = labels
target_res["img_scale"] = torch.tensor([1.0] * batch_size, dtype=torch.float ).to(self.device)
 target_res["img_size"] = torch.tensor([images[0].shape[-2:]] * batch_size, dtype=torch.float ).to(self.device)
 outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 return calc_loss
def save_model(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_calc_loss': self.best_calc_loss,
'epoch': self.epoch,
}, path)
 def load_model(self, path):
model = torch.load(path)
 self.model.load_state_dict(model['model_state_dict'])
 self.optimizer.load_state_dict(model['optimizer_state_dict'])
 self.scheduler.load_state_dict(model['scheduler_state_dict'])
 self.best_calc_loss = model['best_calc_loss']
self.epoch = model['epoch'] + 1
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
def setDomain(df):
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
 return df
train=setDomain(train)
 test=setDomain(test )<feature_engineering>
def get_model(num_classes = 1):
config = get_efficientdet_config('tf_efficientdet_d5')
 model = EfficientDet(config, pretrained_backbone=False)
 checkpoint = torch.load('/content/drive/My Drive/Global Wheat Detection/efficientdet_d7/efficientdet_d5-ef44aea8.pth')
 model.load_state_dict(checkpoint)
 config.num_classes = num_classes
config.image_size = 512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
return DetBenchTrain(model, config )
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"] == "samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"] == "opera 53.0",'lastest_browser']=1
df.loc[df["id_31"] == "mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"] == "google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"] == "firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"] == "edge 17.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for ios",'lastest_browser']=1
return df
train = setBrowser(train)
 test = setBrowser(test )<feature_engineering>
class GlobalParametersTrain:
lr = 0.0002
n_epochs = 40
folder = '/content/drive/My Drive/Global Wheat Detection/efficientdet_d5'
verbose = True
verbose_step = 10
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(mode='min',factor=0.5,patience=1,verbose=False, threshold=0.0001,threshold_mode='abs',cooldown=0, min_lr=1e-8,eps=1e-08 )
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df.loc[df.device_name.isin(df.device_name.value_counts() [df.device_name.value_counts() < 200].index), 'device_name'] = "Others"
df['had_id'] = 1
gc.collect()
return df
train = setDevice(train)
 test = setDevice(test )<concatenate>
def load_model(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 model = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchPredict(model, config)
 model.eval() ;
return model.cuda()
model = load_model('.. /input/efficientdetd5model17e/efficient-best-017epoch.bin' )
i_cols = ['card1','card2','card3','card5',
'C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D8',
'addr1','addr2',
'dist1',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','device_name',
'id_30','id_33',
'uid1','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid1']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']<statistical_test>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def make_predictions(images , score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
outputs = model(images , torch.tensor([1.0] * images.shape[0], dtype=torch.float ).to(device), torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes],
})
 return [predictions]
def get_too_many_null_attr(data):
many_null_cols = [col for col in data.columns if data[col].isnull().sum() / data.shape[0] > 0.9]
return many_null_cols
def get_too_many_repeated_val(data):
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
return big_top_value_cols
def get_useless_columns(data):
too_many_null = get_too_many_null_attr(data)
 print("More than 90% null: " + str(len(too_many_null)))
 too_many_repeated = get_too_many_repeated_val(data)
 print("More than 90% repeated value: " + str(len(too_many_repeated)))
 cols_to_drop = list(set(too_many_null + too_many_repeated))
return cols_to_drop<categorify>
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
for f in train.columns:
if train[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))<categorify>
for j,(images, targets , image_ids)in enumerate(valid_loader):
break
predictions = make_predictions(images)
 i = 0
sample = images[i].permute(1,2,0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=511 )
train = train.fillna(-999)
 test = test.fillna(-999 )<count_missing_values>
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
print(train.isnull().sum().max())
 print(test.isnull().sum().max() )<drop_column>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None], [TTAVerticalFlip() , None],[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
X = train.drop(['TransactionID', 'TransactionDT', 'DT'], axis=1)
 X_test = test.drop(['TransactionID', 'TransactionDT', 'DT'], axis=1 )<init_hyperparams>
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda() , torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
'labels': labels[indexes],
})
 predictions.append(result)
 return predictions
params = {'num_leaves': 546,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.1797454081646243,
'bagging_fraction': 0.2181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.005883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3299927210061127,
'reg_lambda': 0.3885237330340494,
'random_state': SEED,
}<prepare_x_and_y>
test_transforms = Compose([
Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(
 params,
dtrain,
10000,
valid_sets = [dtrain, dvalid],
verbose_eval=200,
early_stopping_rounds=500 
)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<prepare_output>
class TestDataset(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[2]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self):
return self.image_ids.shape[0]
sample_submission['isFraud'] = y_preds<save_to_csv>
test_set = TestDataset(image_ids=np.array([path.split('/')[-1][:-4] for path in test_paths]),transforms=test_transforms)
 test_loader = DataLoader(test_set,batch_size=4,shuffle=False,num_workers=2,drop_last=False,collate_fn=collate_fn )
sample_submission.to_csv('submission.csv', index=False )<save_to_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
<import_modules><EOS> 
SUBMISSION_PATH = '/kaggle/working'
submission_id = 'submission'
submission_path = os.path.join(SUBMISSION_PATH, '{}.csv'.format(submission_id))
sample_submission = pd.DataFrame(submission, columns=["image_id","PredictionString"])
 sample_submission.to_csv(submission_path, index=False)
 submission_df = pd.read_csv(submission_path)
 submission_df