Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<save_to_csv>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('fivethirtyeight')
 print(os.listdir(".. /input"))
SUBMISSION_PATH = '/kaggle/working'
submission_id = 'submission'
submission_path = os.path.join(SUBMISSION_PATH, '{}.csv'.format(submission_id))
sample_submission = pd.DataFrame(submission, columns=["image_id","PredictionString"])
 sample_submission.to_csv(submission_path, index=False)
 submission_df = pd.read_csv(submission_path)
 submission_df<install_modules>
def load_df(csv_path='.. /input/ga-customer-revenue-prediction/train.csv', nrows=None):
COLUMNS_HAVING_JSON = ['device', 'geoNetwork', 'totals', 'trafficSource']
df_json_columns = pd.read_csv(csv_path,
converters={column: json.loads for column in COLUMNS_HAVING_JSON},
dtype={'fullVisitorId': 'str'},
nrows=nrows)
 for column in COLUMNS_HAVING_JSON:
df_column = json_normalize(df_json_columns[column])
 df_column.columns = [f"{column}_{key}" for key in df_column.columns]
df_json_columns = df_json_columns.drop(column, axis=1)
 df_json_columns = df_json_columns.merge(df_column, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df_json_columns.shape}")
 return df_json_columns
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
training_df = load_df(".. /input/ga-customer-revenue-prediction/train.csv")
 test_df = load_df(".. /input/ga-customer-revenue-prediction/test.csv" )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
training_df.to_feather('train_data.feather')
 test_df.to_feather('test_data.feather')
 print("Loaded data in feather file" )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
df = pd.read_feather('train_data.feather')
 test_df = pd.read_feather('test_data.feather')
 print("Loaded from feather file" )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
constant_valued_columns = [c for c in df.columns if df[c].nunique(dropna=False)==1]
non_relevant = ["visitNumber", "date", "fullVisitorId", "sessionId", "visitId", "visitStartTime"]
df = df.drop(constant_valued_columns, axis=1)
 test_df = test_df.drop(constant_valued_columns, axis=1)
 df=df.drop(['trafficSource_campaignCode'],axis=1 )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/effdetd5/best-checkpoint-027epoch.bin' )<categorify>
def convert_totals_columns_to_float(df,isTrain=True):
numeric_cols_to_transform = ['totals_hits'
,'totals_pageviews'
,'totals_newVisits'
,'totals_bounces']
if isTrain == True:
df[numeric_cols_to_transform] = df[numeric_cols_to_transform].fillna(0)
 df['totals_transactionRevenue'] = df['totals_transactionRevenue'].fillna(0)
 else:
df[numeric_cols_to_transform] = df[numeric_cols_to_transform].fillna(0)
 for col in numeric_cols_to_transform:
df[col] = df[col].astype('float32')
 if isTrain == True:
df['totals_transactionRevenue'] = df['totals_transactionRevenue'].astype('float32')
 return df
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
df['totals_transactionRevenue'] = df['totals_transactionRevenue'].astype('float32')
 df=df.drop(['trafficSource_campaignCode'],axis=1 )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
categorical_columns = [c for c in df.columns if not c.startswith("total")]
categorical_columns = [c for c in categorical_columns if c not in constant_valued_columns + non_relevant]
for c in categorical_columns:
le = LabelEncoder()
train_vals = list(df[c].values.astype(str))
test_vals = list(test_df[c].values.astype(str))
le.fit(train_vals + test_vals)
 df[c] = le.transform(train_vals)
 test_df[c] = le.transform(test_vals )
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
df = convert_totals_columns_to_float(df)
 test_df = convert_totals_columns_to_float(test_df,False )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
def normalize_numerical_columns(df, isTrain = True):
df["totals_hits"] = df["totals_hits"].astype(float)
 df["totals_hits"] =(df["totals_hits"] - min(df["totals_hits"])) /(max(df["totals_hits"])- min(df["totals_hits"]))
df["totals_pageviews"] = df["totals_pageviews"].astype(float)
 df["totals_pageviews"] =(df["totals_pageviews"] - min(df["totals_pageviews"])) /(max(df["totals_pageviews"])- min(df["totals_pageviews"]))
if isTrain:
df["totals_transactionRevenue"] = df["totals_transactionRevenue"].fillna(0.0)
 return df
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
df = normalize_numerical_columns(df)
 test_df = normalize_numerical_columns(test_df, isTrain = False )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
features = [c for c in df.columns if c not in constant_valued_columns + non_relevant]
features.remove("totals_transactionRevenue")
 df_new["totals_transactionRevenue"] = np.log1p(df["totals_transactionRevenue"].astype(float))
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
df_count_nonzero_tr = df.groupby('fullVisitorId')['totals_transactionRevenue'].agg(['count'])
 df_count_nonzero_tr.columns = ["Count"]
totalNonZeroTransactionCount = df_count_nonzero_tr["Count"].sum()
df_count_nonzero_tr['buying_probability'] = df_count_nonzero_tr["Count"]/totalNonZeroTransactionCount
df_count_nonzero_tr=df_count_nonzero_tr.sort_values(by=['buying_probability'],ascending=False)
 print(df_count_nonzero_tr.head(10))
probSum = df_count_nonzero_tr['buying_probability'].sum()
print(probSum )
USE_OPTIMIZE = True
USE_TTA = True<choose_model_class>
train_x, valid_x, train_y, valid_y = train_test_split(df[features], df_new["totals_transactionRevenue"], test_size=0.20, random_state=42 )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint["model_state_dict"])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
if USE_OPTIMIZE:
models = [
load_net('.. /input/effdet-fold0/best-checkpoint-023epoch.bin'),
load_net('.. /input/effdet-fold1/best-checkpoint-017epoch.bin')]<define_variables>
lgb_params = {"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 200,
"min_child_samples" : 100,
"learning_rate" : 0.05,
"boosting_type" : "gbdt",
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1}
lgb_train = lgb.Dataset(train_x, label=train_y)
 lgb_val = lgb.Dataset(valid_x, label=valid_y)
 model = lgb.train(lgb_params, lgb_train, 700, valid_sets=[lgb_val], early_stopping_rounds=150 )
best_final_score = 0.7202
best_iou_thr = 0.460
best_skip_box_thr = 0.394<normalization>
unimportant_features = ["visitNumber", "date", "fullVisitorId", "sessionId", "visitId", "visitStartTime"]
features = [c for c in df.columns if c not in constant_valued_columns + unimportant_features]
print(features )
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes )<data_type_conversions>
features = [c for c in df.columns if c not in constant_valued_columns + unimportant_features]
features.remove("totals_transactionRevenue")
 preds = model.predict(test_df[features], num_iteration=model.best_iteration)
 test_df["PredictedLogRevenue"] = np.expm1(preds)
 result = test_df.groupby("fullVisitorId" ).agg({"PredictedLogRevenue" : "sum"} ).reset_index()
result["PredictedLogRevenue"] = np.log1p(result["PredictedLogRevenue"])
 result["PredictedLogRevenue"] = result["PredictedLogRevenue"].apply(lambda x : 0.0 if x < 0 else x)
 result["PredictedLogRevenue"] = result["PredictedLogRevenue"].fillna(0)
 result.to_csv("baseline.csv", index=False)
 print(result.isnull().values.any())
 result.head()
def to_tensor(images):
tmp = []
for img in images:
img = img.astype(np.float32)
 img /= 255.0
img = torch.tensor(img, dtype=torch.float32)
 tmp.append(img.permute(2,0,1))
return torch.stack(tmp )<compute_train_metric>
def calculate_rmse(df_new):
temp_df = pd.DataFrame()
temp_df["totals_transactionRevenue"] = np.log1p(df_new["totals_transactionRevenue"].astype(float))
train_x, valid_x, train_y, valid_y = train_test_split(df_new[features], temp_df["totals_transactionRevenue"], test_size=0.2, random_state=20)
 lgb_params = {"objective" : "regression", "metric" : "rmse",
"num_leaves" : 50, "learning_rate" : 0.02,
"bagging_fraction" : 0.75, "feature_fraction" : 0.8, "bagging_frequency" : 9}
lgb_train = lgb.Dataset(train_x, label=train_y)
 lgb_val = lgb.Dataset(valid_x, label=valid_y)
 model = lgb.train(lgb_params, lgb_train, 400, valid_sets=[lgb_val], early_stopping_rounds=100, verbose_eval=200)
 prediction = model.predict(valid_x, num_iteration=model.best_iteration)
 rmse = np.sqrt(metrics.mean_squared_error(prediction, valid_y))
return rmse
def run_wbf_tta(predictions, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes_1 = []
scores_1 = []
labels_1 = []
for i in range(2):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_1.append(bb)
 scores_1.append(ss)
 labels_1.append(ls)
 boxes_1, scores_1, labels_1 = weighted_boxes_fusion(boxes_1, scores_1, labels_1,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes_2 = []
scores_2 = []
labels_2 = []
for i in range(2,4):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_2.append(bb)
 scores_2.append(ss)
 labels_2.append(ls)
 boxes_2, scores_2, labels_2 = weighted_boxes_fusion(boxes_2, scores_2, labels_2,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = [boxes_1.tolist() , boxes_2.tolist() ]
scores = [scores_1.tolist() , scores_2.tolist() ]
labels = [labels_1.tolist() , labels_2.tolist() ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
testing_cols = ['trafficSource_campaign','totals.pageviews','trafficSource.adContent']
for cols in testing_cols:
df_new = df.copy()
count = 0
i=1
while i < 500:
df_new[cols] = np.random.permutation(df_new[cols])
 if calculate_rmse(df_new)<bestValue:
count=count+1
print(count)
 print("For ",cols," number of times RMSE is lesser than base is : ",count )