Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<init_hyperparams>
!cp '.. /input/weightedboxesfusion'.-r
!pip install --no-deps './weightedboxesfusion' > /dev/null
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}<train_model>
np.random.seed(123)
 torch.manual_seed(123 )
folds = TimeSeriesSplit(n_splits=5)
 aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
training_start_time = time()
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, y)) :
start_time = time()
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 print('Fold {} finished in {}'.format(fold + 1, str(datetime.timedelta(seconds=time() - start_time))))
print('-' * 30)
 print('Training has finished.')
 print('Total training time is {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))
print('Mean AUC:', np.mean(aucs))
print('-' * 30 )<save_to_csv>
N_CLUSTERS = 1
PL_ROUNDS = 5
PL_EPOCHS = 5
detection_threshold = 0.5
image_size = 1024
feature_importances['average'] = feature_importances[['fold_{}'.format(fold + 1)for fold in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances_TimeFold.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;<find_best_params>
dataset_path = '/kaggle/input/global-wheat-detection'
original_train_path = '/kaggle/input/original-train/original_train'
weights_path = '/kaggle/input/manual-weights-7/e152.pth'
train_path = join(dataset_path, 'train')
 test_path = join(dataset_path, 'test')
 train_df = pd.read_csv(join(original_train_path, 'train.csv'))
train_df['image_path'] = [join(original_train_path, s, i + '.jpeg')for i, s in train_df[['image_id', 'source']].values]
test_df = pd.read_csv(join(dataset_path, 'sample_submission.csv'))
best_iter = clf.best_iteration<train_model>
id_to_path = {}
for idx, row in train_df.iterrows() :
image_id = row['image_id']
if image_id in id_to_path.keys() : continue
id_to_path[image_id] = row['image_path']
for idx, row in test_df.iterrows() :
image_id = row['image_id']
if image_id in id_to_path.keys() : continue
id_to_path[image_id] = join(test_path, image_id + '.jpg' )
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, y )<save_to_csv>
class WheatTrainDataset(Dataset):
def __init__(self, dataframe, transforms=None, test=False):
super().__init__()
self.df = dataframe
self.image_ids = pd.unique(dataframe['image_id'])
 self.image_paths = [id_to_path[image_id] for image_id in self.image_ids]
self.length = len(self.image_ids)
 self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image_path = self.image_paths[index]
image = self.load_image(image_path)
 boxes = self.load_boxes(image_id)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {'boxes': boxes, 'labels': labels, 'image_id': torch.tensor([index])}
if self.transforms:
sample = {'image': image, 'bboxes': target['boxes'], 'labels': labels}
sample = self.transforms(**sample)
 image, boxes = sample['image'], sample['bboxes']
boxes = self.filter_boxes(boxes)
 if len(boxes):
target['boxes'] = torch.stack([torch.tensor(box, dtype=torch.float32)for box in zip(*boxes)] ).permute(1, 0)
 else:
return self.__getitem__(np.random.randint(self.length))
return image, target, image_id
def __len__(self)-> int:
return self.length
def load_image(self, image_path):
image = cv2.imread(image_path , cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
return image
def load_boxes(self, image_id):
records = self.df[self.df['image_id'] == image_id]
if 'x1' in records.columns and 'y1' in records.columns:
return records[['x', 'y', 'x1', 'y1']].values
else:
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return boxes
def filter_boxes(self, boxes):
min_length = 13
min_area = 400
max_area = 145360
max_length_ratio = 18
min_length_ratio = 1.0/max_length_ratio
boxes_out = []
for box in boxes:
x, y, x1, y1 = box
w = x1 - x
if w < min_length: continue
h = y1 - y
if h < min_length: continue
area = w * h
if area < 400 or area > 145360: continue
length_ratio = w / h
if length_ratio < min_length_ratio or length_ratio > max_length_ratio: continue
boxes_out.append(box)
 return boxes_out
def get_sample_weights(self):
weights = []
for image_id in self.image_ids:
w, h, source = self.df[self.df['image_id'] == image_id][['width', 'height', 'source']].values[0]
if w == 3072:
weight = 8
elif h == 2048:
weight = 5
elif w == 2048:
weight = 2.5
else:
weight = 1
if source == 'test': weight *= 10
weights.append(weight)
 return np.array(weights)
 class WheatTestDataset(Dataset):
def __init__(self, dataframe, image_dir, image_size=1024, onfly=False, tta=True):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.image_size = image_size
self.onfly = onfly
self.tta = tta
def load_image(self, image_id):
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = cv2.resize(image,(self.image_size, self.image_size))
image = image.astype(np.float32)
 image /= 255.0
return image
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = self.load_image(image_id)
 if self.tta:
all_images = apply_tta(image)
 return all_images, image_id
else:
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def collate_fn(batch): return tuple(zip(*batch))
predictions = clf.predict_proba(test)[:, 1]
output = pd.DataFrame({ "TransactionID" : test_TransactionID, "isFraud": predictions })
 output.to_csv('submission_lgbm.csv', index=False)
<import_modules>
def run_wbf(predictions, image_size=1024, iou_thr=0.4, skip_box_thr=0.7, weights=None):
boxes = [pred['boxes'].data.cpu().numpy() for pred in predictions]
boxes = revert_tta(boxes, image_size)
 boxes = [box_set/(image_size-1)for box_set in boxes]
scores = [pred['scores'].data.cpu().numpy() for pred in predictions]
labels = [np.ones(pred['scores'].shape[0])for pred in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
from scipy.stats import spearmanr, rankdata
import pandas as pd
import numpy as np
import pickle<load_from_csv>
def get_resnet() :
backbone = resnet.__dict__['resnet50'](pretrained=False, norm_layer=FrozenBatchNorm2d)
 for name, parameter in backbone.named_parameters() :
if 'layer2' not in name and 'layer3' not in name and 'layer4' not in name:
parameter.requires_grad_(False)
 return backbone
class MyFasterRCNN(nn.Module):
def __init__(self):
super(MyFasterRCNN, self ).__init__()
image_mean = [0.485, 0.456, 0.406]
image_std = [0.229, 0.224, 0.225]
self.transform = GeneralizedRCNNTransform(1024, 1024, image_mean, image_std)
 self.backbone = get_resnet()
return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
self.body = IntermediateLayerGetter(self.backbone, return_layers=return_layers)
 out_channels = 256
in_channels_list = [256 *(2 ** i)for i in range(4)]
self.fpn = FeaturePyramidNetwork(
 in_channels_list=in_channels_list,
out_channels=out_channels,
extra_blocks=LastLevelMaxPool())
 anchor_sizes =(( 32,),(64,),(128,),(256,),(512,))
aspect_ratios =(( 0.5, 1.0, 2.0),)* len(anchor_sizes)
 anchor_generator = AnchorGenerator(anchor_sizes, aspect_ratios)
 head = RPNHead(out_channels, anchor_generator.num_anchors_per_location() [0])
 self.rpn = RegionProposalNetwork(
 anchor_generator=anchor_generator,
head=head,
fg_iou_thresh=0.7,
bg_iou_thresh=0.3,
batch_size_per_image=256,
positive_fraction=0.5,
pre_nms_top_n=dict(training=2000, testing=1000),
post_nms_top_n=dict(training=2000, testing=1000),
nms_thresh=0.7)
 representation_size = 512
box_roi_pool = MultiScaleRoIAlign(['0', '1', '2', '3'], 7, 2)
 box_head = TwoMLPHead(out_channels * box_roi_pool.output_size[0] ** 2, representation_size)
 box_predictor = FastRCNNPredictor(representation_size, num_classes=2)
 self.roi_heads = RoIHeads(
 box_roi_pool=box_roi_pool,
box_head=box_head,
box_predictor=box_predictor,
fg_iou_thresh=0.5, bg_iou_thresh=0.5,
batch_size_per_image=512, positive_fraction=0.25,
bbox_reg_weights=None,
score_thresh=0.05,
nms_thresh=0.5,
detections_per_img=100)
 def forward(self, images, targets=None):
images, targets = self.transform(images, targets)
 features = self.body(images.tensors)
 fpn_features = self.fpn(features)
 if isinstance(fpn_features, torch.Tensor): fpn_features = OrderedDict([('0', fpn_features)])
 proposals, proposal_losses = self.rpn(images, fpn_features, targets)
 detections, detector_losses = self.roi_heads(fpn_features, proposals, images.image_sizes, targets)
 losses = {}
losses.update(detector_losses)
 losses.update(proposal_losses)
 return features, detections, losses
class ResNetFeaturizer(nn.Module):
def __init__(self):
super(ResNetFeaturizer, self ).__init__()
image_mean = [0.485, 0.456, 0.406]
image_std = [0.229, 0.224, 0.225]
self.transform = GeneralizedRCNNTransform(1024, 1024, image_mean, image_std)
 self.backbone = get_resnet()
self.backbone.load_state_dict(torch.load('/kaggle/input/pretrained-pytorch/resnet50-19c8e357.pth'))
def forward(self, images, targets=None):
images, targets = self.transform(images, targets)
 features = self.backbone(images.tensors)
 return features
data_folder = ".. /input/ieee-fraud-detection/"
sub = pd.read_csv(f'{data_folder}sample_submission.csv' )<load_pretrained>
def get_model(weights_path):
model = MyFasterRCNN()
num_classes = 2
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 model.load_state_dict(torch.load(weights_path))
model = model.cuda()
model = model.eval()
return model
model = get_model(weights_path )
with open(".. /input/ieee-sol1/test_uid_nocoverage.p", "rb")as f:
test_uid_nocoverage = pickle.load(f )<load_from_csv>
test_dataset = WheatTestDataset(test_df, test_path, tta=False)
 test_data_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=4, drop_last=False, collate_fn=collate_fn )
sub1 = rankdata(pd.read_csv(".. /input/ieee-submissions-and-uids/final_model_blend.csv" ).isFraud)
 sub1 /= len(sub1)
 sub2 = rankdata(pd.read_csv(".. /input/ieee-sol1/blend_of_blends_1.csv" ).isFraud)
 sub2 /= len(sub2 )<compute_test_metric>
def cluster_test_data() :
rn_model = ResNetFeaturizer().cuda().eval()
feature_vecs = []
image_ids = []
for images, image_id in tqdm(test_data_loader):
inp = [torch.tensor(images[0] ).cuda().permute(2,0,1)]
fv = rn_model(inp)
 feature_vecs.append(fv.detach().cpu().numpy())
 image_ids.append(image_id[0])
 feature_vecs = np.squeeze(np.array(feature_vecs))
print(feature_vecs.shape)
 agglo = AgglomerativeClustering(n_clusters=N_CLUSTERS)
 clusters = agglo.fit_predict(feature_vecs)
 id_to_cluster = {k : v for k,v in zip(image_ids, clusters)}
del rn_model
del inp
del fv
torch.cuda.empty_cache()
return id_to_cluster
spearmanr(sub1, sub2 )<feature_engineering>
id_to_cluster = cluster_test_data()
sub3 = sub1
sub3[test_uid_nocoverage] = 0.5 * sub1[test_uid_nocoverage] + 0.5 * sub2[test_uid_nocoverage]<save_to_csv>
test_df['cluster_id'] = [id_to_cluster[iid] for iid in test_df['image_id'].values]
sub['isFraud'] = sub3
sub.to_csv('submission.csv', index=False)
 sub.head()<install_modules>
def get_pseudo_labels(model, data_loader, combine_with_train=False):
model.eval()
data = []
for images, image_ids in data_loader:
predictions = []
for img in images[0]:
inp = [torch.tensor(img ).cuda().permute(2,0,1)]
features, pred, loss = model(inp)
 predictions.extend(pred)
 boxes, scores, labels = run_wbf(predictions, image_size=image_size)
 boxes =(boxes ).astype(np.int32 ).clip(min=0, max=int(image_size-1))
image_id = image_ids[0]
for box in boxes:
x0, y0, x1, y1 = box
w = x1 - x0
h = y1 - y0
data.append([image_id, 1024, 1024, 'test', x0, y0, w, h, x1, y1, image_id, join(test_path, image_id + '.jpg'), id_to_cluster[image_id]])
 s1_df = pd.DataFrame(data, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h', 'x1', 'y1', 'image_id_orig', 'image_path', 'cluster_id'])
 if combine_with_train:
s1_df = pd.concat([s1_df, train_df])
 return s1_df
%%time
!cp.. /input/rapids/rapids.0.11.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz
sys.path = ["/opt/conda/envs/rapids/lib"] + ["/opt/conda/envs/rapids/lib/python3.6"] + ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/<set_options>
def train_round(round_num, model):
for cluster_id in range(N_CLUSTERS):
if round_num == 0:
model.load_state_dict(torch.load(weights_path))
else:
model.load_state_dict(torch.load('round_%d_cluster_%d.pth' %(round_num-1, cluster_id)))
 optimizer = torch.optim.SGD(model.parameters() , lr=0.001, momentum=0.9, weight_decay=0.0001)
 cluster_df = test_df[test_df['cluster_id'] == cluster_id]
cluster_dataset = WheatTestDataset(cluster_df, test_path)
 cluster_data_loader = DataLoader(cluster_dataset, batch_size=1, shuffle=False, num_workers=4, drop_last=False, collate_fn=collate_fn)
 cluster_df = get_pseudo_labels(model, cluster_data_loader)
 train_dataset = WheatTrainDataset(cluster_df, get_train_transforms())
 train_data_loader = DataLoader(train_dataset, batch_size=8, num_workers=4, drop_last=False, collate_fn=collate_fn)
 for epoch in range(PL_EPOCHS):
print('Round %d | Cluster %d | Epoch %d' %(round_num, cluster_id, epoch))
model.train()
for step,(images, targets, image_ids)in enumerate(train_data_loader):
images = [img.cuda() for img in images]
targets = [{k: v.cuda() for k, v in l.items() } for l in targets]
features, detections, losses = model(images, targets)
 optimizer.zero_grad()
loss_FS = sum(losses.values())
 loss_FS.backward()
optimizer.step()
torch.save(model.state_dict() , 'round_%d_cluster_%d.pth' %(round_num, cluster_id))
warnings.filterwarnings("ignore")
 LOCAL_VALIDATION = False
cols = ['TransactionID', 'TransactionDT', 'TransactionAmt',
'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8',
'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4',
'M5', 'M6', 'M7', 'M8', 'M9']
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
cols += ['V'+str(x)for x in v]
dtypes = {'isFraud':'int8'}
for c in cols+['id_0'+str(x)for x in range(1,10)]+['id_'+str(x)for x in range(10,34)]: dtypes[c] = 'float32'
for c in ['id-0'+str(x)for x in range(1,10)]+['id-'+str(x)for x in range(10,34)]: dtypes[c] = 'float32'
str_type = ['ProductCD', 'card4', 'card6', 'P_emaildomain', 'R_emaildomain','M1', 'M2', 'M3', 'M4','M5',
'M6', 'M7', 'M8', 'M9', 'id_12', 'id_15', 'id_16', 'id_23', 'id_27', 'id_28', 'id_29', 'id_30',
'id_31', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
str_type += ['id-12', 'id-15', 'id-16', 'id-23', 'id-27', 'id-28', 'id-29', 'id-30',
'id-31', 'id-33', 'id-34', 'id-35', 'id-36', 'id-37', 'id-38']
for c in str_type: dtypes[c] = 'category'
start = time.time()
print('RAPIDS =',cudf.__version__ )<load_from_csv>
for round_num in range(PL_ROUNDS):
train_round(round_num, model )
%%time
X_train = cudf.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv',index_col='TransactionID', usecols=cols+['isFraud'],dtype=dtypes)
 train_id = cudf.read_csv('.. /input/ieee-fraud-detection/train_identity.csv',index_col='TransactionID',dtype=dtypes)
 X_train = X_train.merge(train_id, how='left', left_index=True, right_index=True)
 X_test = cudf.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv',index_col='TransactionID', usecols=cols,dtype=dtypes)
 test_id = cudf.read_csv('.. /input/ieee-fraud-detection/test_identity.csv',index_col='TransactionID', dtype=dtypes)
 fix = {o:n for o, n in zip(test_id.columns, train_id.columns)}
test_id.rename(columns=fix, inplace=True)
 X_test = X_test.merge(test_id, how='left', left_index=True, right_index=True)
 del train_id, test_id; x = gc.collect()
print('Train shape on GPU',X_train.shape,'Test shape on GPU',X_test.shape )<categorify>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
%%time
for i in range(1,16):
if i in [1,2,3,5,9]: continue
X_train['D'+str(i)] = X_train['D'+str(i)] - X_train.TransactionDT/np.float32(24*60*60)
 X_test['D'+str(i)] = X_test['D'+str(i)] - X_test.TransactionDT/np.float32(24*60*60)
 def encode_LE(df1,df2,col,verbose=True):
df_comb = cudf.concat([df1[col],df2[col]],axis=0)
 df_comb,_ = df_comb.factorize()
df1[col] = df_comb[:len(df1)].astype('int32')
 df2[col] = df_comb[len(df1):].astype('int32')
 if verbose: print(col,', ',end='')
 for i,f in enumerate(X_train.columns):
if(X_train[f].dtype=='object'):
encode_LE(X_train,X_test,f,False)
 elif f in str_type:
X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True)
 elif f not in ['TransactionAmt','TransactionDT','isFraud']:
mn = np.min(( X_train[f].min() ,X_test[f].min()))
X_train[f] -= np.float32(mn)
 X_test[f] -= np.float32(mn)
 X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True )<feature_engineering>
detection_threshold = 0.5
image_size = 1024
results = []
for cluster_id in range(N_CLUSTERS):
model.load_state_dict(torch.load('round_%d_cluster_%d.pth' %(round_num, cluster_id)))
 model.eval()
test_df_cluster = test_df[test_df['cluster_id'] == cluster_id]
test_dataset = WheatTestDataset(test_df_cluster, test_path)
 test_data_loader = DataLoader(test_dataset, batch_size=1, num_workers=4, drop_last=False, collate_fn=collate_fn)
 for images, image_ids in test_data_loader:
image_id = image_ids[0]
predictions = []
for img in images[0]:
inp = [torch.tensor(img ).cuda().permute(2,0,1)]
features, pred, loss = model(inp)
 predictions.extend(pred)
 boxes, scores, labels = run_wbf(predictions, image_size=image_size)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=int(image_size-1))
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
def add_feature(df1,df2,uid,col,agg,verbose=True):
if agg=='count': func = count2
elif agg=='mean': func = mean2
elif agg=='std' : func = std2
elif agg=='nunique': func = nunique2
else: return
df1['idx'] = np.arange(len(df1))
df2['idx'] = np.arange(len(df2)) +len(df1)
 temp_df = cudf.concat([df1[[uid,col,'idx']], df2[[uid,col,'idx']]])
 tmp = temp_df.groupby(uid,method='cudf' ).apply_grouped(
 func,
incols={col:'x'},
outcols=dict(y_out=np.float32),
tpb=32
).rename({'y_out':'new'})
 tmp = tmp.sort_values('idx')
 df1[uid+'_'+col+'_'+agg] = tmp.iloc[:len(df1)].new
df2[uid+'_'+col+'_'+agg] = tmp.iloc[len(df1):].new
if verbose: print(uid+'_'+col+'_'+agg,', ',end='')
 df1.drop_column('idx')
 df2.drop_column('idx')
 def add_features(df1,df2,uids,cols,aggs,verbose=True):
for col in cols:
for uid in uids:
for agg in aggs:
add_feature(df1,df2,uid,col,agg,verbose)
 def encode_CB(df1,df2,col1,col2,verbose=True):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(df1,df2,nm,verbose=False)
 if verbose: print(nm,', ',end='' )<categorify>
sub_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 sub_df.head()
<feature_engineering><EOS> 
sub_df.to_csv('submission.csv', index=False )