Kaggle competition: Google Analytics Customer Revenue Prediction
results = []
for images, image_ids in data_loader:
if USE_TTA:
image = images[0]
predictions_tta = {
"boxes": [],
"scores": [],
"labels": []
}
for index in range(4):
roated = TTAImage(image, index)
 roated = to_tensor([roated] ).cuda()
predictions = make_predictions(roated)
 boxes, scores, labels = run_wbf(predictions, image_index=0)
 for _ in range(3-index):
boxes = rotBoxes90(boxes, 512, 512)
 if index == 3:
boxes = boxes.astype(np.int32)
 predictions_tta["boxes"].append(boxes)
 predictions_tta["scores"].append(scores)
 predictions_tta["labels"].append(labels)
 boxes, scores, labels = run_wbf_tta(predictions_tta)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_ids[0],
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 else:
images = to_tensor(images ).cuda()
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestRegressor as RFF
import xgboost as xgb
from pprint import pprint
import math
from scipy.stats import kurtosis, skew
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<install_modules>
plt.rcParams['figure.figsize'] =(12,6 )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<load_from_csv>
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<categorify>
print(f'
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
def collate_fn(batch):
return tuple(zip(*batch))
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes<load_pretrained>
def rmse(y_true, y_pred):
return round(np.sqrt(mean_squared_error(y_true, y_pred)) , 5)
 def split_data(df=df_train,rate=.8):
df = df.sort_values('date' ).copy()
df.drop(['fullVisitorId','visitId','visitStartTime'],axis=1,inplace=True)
 df['Revenue'] = np.log1p(df['Revenue'])
 global X_train,X_valid,y_train,y_valid
n_train = int(len(df)*rate)
 X_train = df.drop(['Revenue','date'],axis=1 ).iloc[:n_train]
X_valid = df.drop(['Revenue','date'],axis=1 ).iloc[n_train:]
y_train = df['Revenue'].iloc[:n_train]
y_valid = df['Revenue'].iloc[n_train:]
print(X_train.shape,X_valid.shape)
 def encode_data(verbose=False):
global df_train_encoded,df_test_encoded
df_train_encoded = df_train.copy()
df_test_encoded = df_test.copy()
for col in df_train.columns:
if df_train_encoded[col].dtype == 'object' and col not in ['fullVisitorId','visitId','visitStartTime','date']:
if verbose:
print(col)
 lb = LabelEncoder()
lb.fit(list(df_train_encoded[col].unique())+ list(df_test_encoded[col].unique()))
df_train_encoded[col] = lb.transform(df_train_encoded[col])
 df_test_encoded[col] = lb.transform(df_test_encoded[col])
 def run_xgb() :
params = {
'objective':'reg:linear',
'eval_metric':'rmse',
'learning_rate':.01,
'eta': 0.15,
'subsample': 0.6,
'colsample_bytree': 0.6,
'lambda':1,
'random_state': 42,
'silent':True
}
params['n_thread'] = -1
params['max_depth'] = 8
params['min_child_weight'] = 100
params['gamma'] = 5
params['subsample'] = 1
params['colsample_bytree'] =.95
params['colsample_bylevel'] = 0.35
params['alpha'] = 25
params['lambda'] = 25
xgb_train_data = xgb.DMatrix(X_train, y_train)
 xgb_val_data = xgb.DMatrix(X_valid, y_valid)
 model = xgb.train(params, xgb_train_data,
num_boost_round = 200,
evals= [(xgb_train_data, 'train'),(xgb_val_data, 'valid')],
early_stopping_rounds=50,
verbose_eval=20 
)
 return model
def submit() :
test_matrix = xgb.DMatrix(X_test)
 y_pred = clf.predict(test_matrix,ntree_limit=clf.best_ntree_limit)
 df_test['PredictedLogRevenue'] = y_pred
engineer_prediction
print('rmse after engineer prediction')
 print(rmse(y_pred,df_test['PredictedLogRevenue']))
submit = df_test[['PredictedLogRevenue','fullVisitorId']].groupby('fullVisitorId' ).PredictedLogRevenue.sum().reset_index()
submit.to_csv('submit.csv',index=False)
 test(y_pred)
 def engineer_prediction(df_test):
df_test[df_test['totals_hits'] == 1].PredictedLogRevenue = 0
df_test[df_test['totals_timeOnSite'] == 0].PredictedLogRevenue = 0
df_test[df_test['totals_bouces'] == 1].PredictedLogRevenue = 0
return dftest
def test(predict):
y_test = np.log1p(df_test['totals_transactionRevenue'])
 print(rmse(y_test,predict))
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint, strict=False)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin'),
load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin'),
load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin'),
load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin'),
load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin')
 ]<load_pretrained>
def prepare_data(df_train,df_test,
del_col=['fullVisitorId','visitId','visitStartTime','date'],to_log=None):
df_train = df_train.sort_values('date' ).copy()
df_train = df_train.drop(del_col,axis=1 ).copy()
df_test = df_test.drop(del_col,axis=1 ).copy()
if to_log is not None:
df_train[to_log] = np.log1p(df_train[to_log])
 df_test[to_log] = np.log1p(df_test[to_log])
 df_train['totals_transactionRevenue'] = np.log1p(df_train['totals_transactionRevenue'])
 df_test['totals_transactionRevenue'] = np.log1p(df_test['totals_transactionRevenue'])
 global X_train,X_valid,y_train,y_valid,X_test,y_test
n_train = int(len(df_train)*.8)
 X_train = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[:n_train]
X_valid = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[n_train:]
y_train = df_train['totals_transactionRevenue'].iloc[:n_train]
y_valid = df_train['totals_transactionRevenue'].iloc[n_train:]
X_test = df_test.drop(['totals_transactionRevenue'],axis=1)
 y_test = df_test['totals_transactionRevenue']
%%time
all_predictions = []
for fold_number in range(5):
validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 validation_loader = DataLoader(
 validation_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn 
)
 for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
det = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
all_predictions.append({
'pred_boxes':(boxes*2 ).clip(min=0, max=1023 ).astype(int),
'scores': scores,
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
'image_id': image_ids[i],
} )<normalization>
def feature_engineering(df):
df = df.copy()
df['month'] = df['date'].dt.month
df['day'] = df['date'].dt.day
df['weekday'] = df['date'].dt.weekday
df['weekofyear'] = df['date'].dt.weekofyear
df['browser_category'] = df['device_browser'] + '_' + df['device_deviceCategory']
df['browser_operatingSystem'] = df['device_browser'] + '_' + df['device_operatingSystem']
df['month_unique_user_count'] = df.groupby('month')['fullVisitorId'].transform('nunique')
 df['day_unique_user_count'] = df.groupby('day')['fullVisitorId'].transform('nunique')
 df['weekday_unique_user_count'] = df.groupby('weekday')['fullVisitorId'].transform('nunique')
 df['weekofyear_unique_user_count'] = df.groupby('weekofyear')['fullVisitorId'].transform('nunique')
 df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')
 df['sum_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')
 df['count_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')
 df['mean_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')
 df['mean_hits_per_day'] = df.groupby(['day'])['totals_hits'].transform('mean')
 df['sum_hits_per_day'] = df.groupby(['day'])['totals_hits'].transform('sum')
 df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')
 df['sum_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('mean')
 df['sum_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')
 df['count_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')
 df['mean_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')
 df['sum_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('sum')
 df['count_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('count')
 df['mean_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('mean')
 df['sum_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('sum')
 df['count_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('count')
 df['mean_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('mean')
 df['user_pageviews_sum'] = df.groupby('fullVisitorId')['totals_pageviews'].transform('sum')
 df['user_hits_sum'] = df.groupby('fullVisitorId')['totals_hits'].transform('sum')
 df['user_pageviews_count'] = df.groupby('fullVisitorId')['totals_pageviews'].transform('count')
 df['user_hits_count'] = df.groupby('fullVisitorId')['totals_hits'].transform('count')
 df['user_pageviews_sum_to_mean'] = df['user_pageviews_sum'] / df['user_pageviews_sum'].mean()
df['user_hits_sum_to_mean'] = df['user_hits_sum'] / df['user_hits_sum'].mean()
df['user_pageviews_to_region'] = df['user_pageviews_sum'] / df['mean_pageviews_per_region']
df['user_hits_to_region'] = df['user_hits_sum'] / df['mean_hits_per_region']
return df
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )<compute_train_metric>
df_train = feature_engineering(df_train)
 df_test = feature_engineering(df_test )
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<find_best_params>
encode_data(verbose=True)
 prepare_data(df_train_encoded,df_test_encoded,del_col=['fullVisitorId','visitId',
'visitStartTime','date'],to_log=['visitNumber','totals_hits','totals_pageviews',
'totals_sessionQualityDim','totals_timeOnSite','trafficSource_keyword'])
<compute_test_metric><EOS>