Kaggle competition: Google Analytics Customer Revenue Prediction
def run_wbf(predictions, image_size=1024, iou_thr=0.4, skip_box_thr=0.7, weights=None):
boxes = [pred['boxes'].data.cpu().numpy() for pred in predictions]
boxes = revert_tta(boxes, image_size)
 boxes = [box_set/(image_size-1)for box_set in boxes]
scores = [pred['scores'].data.cpu().numpy() for pred in predictions]
labels = [np.ones(pred['scores'].shape[0])for pred in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels 
<normalization>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline
def get_resnet() :
backbone = resnet.__dict__['resnet50'](pretrained=False, norm_layer=FrozenBatchNorm2d)
 for name, parameter in backbone.named_parameters() :
if 'layer2' not in name and 'layer3' not in name and 'layer4' not in name:
parameter.requires_grad_(False)
 return backbone
class MyFasterRCNN(nn.Module):
def __init__(self):
super(MyFasterRCNN, self ).__init__()
image_mean = [0.485, 0.456, 0.406]
image_std = [0.229, 0.224, 0.225]
self.transform = GeneralizedRCNNTransform(1024, 1024, image_mean, image_std)
 self.backbone = get_resnet()
return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}
self.body = IntermediateLayerGetter(self.backbone, return_layers=return_layers)
 out_channels = 256
in_channels_list = [256 *(2 ** i)for i in range(4)]
self.fpn = FeaturePyramidNetwork(
 in_channels_list=in_channels_list,
out_channels=out_channels,
extra_blocks=LastLevelMaxPool())
 anchor_sizes =(( 32,),(64,),(128,),(256,),(512,))
aspect_ratios =(( 0.5, 1.0, 2.0),)* len(anchor_sizes)
 anchor_generator = AnchorGenerator(anchor_sizes, aspect_ratios)
 head = RPNHead(out_channels, anchor_generator.num_anchors_per_location() [0])
 self.rpn = RegionProposalNetwork(
 anchor_generator=anchor_generator,
head=head,
fg_iou_thresh=0.7,
bg_iou_thresh=0.3,
batch_size_per_image=256,
positive_fraction=0.5,
pre_nms_top_n=dict(training=2000, testing=1000),
post_nms_top_n=dict(training=2000, testing=1000),
nms_thresh=0.7)
 representation_size = 512
box_roi_pool = MultiScaleRoIAlign(['0', '1', '2', '3'], 7, 2)
 box_head = TwoMLPHead(out_channels * box_roi_pool.output_size[0] ** 2, representation_size)
 box_predictor = FastRCNNPredictor(representation_size, num_classes=2)
 self.roi_heads = RoIHeads(
 box_roi_pool=box_roi_pool,
box_head=box_head,
box_predictor=box_predictor,
fg_iou_thresh=0.5, bg_iou_thresh=0.5,
batch_size_per_image=512, positive_fraction=0.25,
bbox_reg_weights=None,
score_thresh=0.05,
nms_thresh=0.5,
detections_per_img=100)
 def forward(self, images, targets=None):
images, targets = self.transform(images, targets)
 features = self.body(images.tensors)
 fpn_features = self.fpn(features)
 if isinstance(fpn_features, torch.Tensor): fpn_features = OrderedDict([('0', fpn_features)])
 proposals, proposal_losses = self.rpn(images, fpn_features, targets)
 detections, detector_losses = self.roi_heads(fpn_features, proposals, images.image_sizes, targets)
 losses = {}
losses.update(detector_losses)
 losses.update(proposal_losses)
 return features, detections, losses
class ResNetFeaturizer(nn.Module):
def __init__(self):
super(ResNetFeaturizer, self ).__init__()
image_mean = [0.485, 0.456, 0.406]
image_std = [0.229, 0.224, 0.225]
self.transform = GeneralizedRCNNTransform(1024, 1024, image_mean, image_std)
 self.backbone = get_resnet()
self.backbone.load_state_dict(torch.load('/kaggle/input/pretrained-pytorch/resnet50-19c8e357.pth'))
def forward(self, images, targets=None):
images, targets = self.transform(images, targets)
 features = self.backbone(images.tensors)
 return features<choose_model_class>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape
def get_model(weights_path):
model = MyFasterRCNN()
num_classes = 2
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 model.load_state_dict(torch.load(weights_path))
model = model.cuda()
model = model.eval()
return model
model = get_model(weights_path )<create_dataframe>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
test_dataset = WheatTestDataset(test_df, test_path, tta=False)
 test_data_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=4, drop_last=False, collate_fn=collate_fn )<find_best_model_class>
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
def cluster_test_data() :
rn_model = ResNetFeaturizer().cuda().eval()
feature_vecs = []
image_ids = []
for images, image_id in tqdm(test_data_loader):
inp = [torch.tensor(images[0] ).cuda().permute(2,0,1)]
fv = rn_model(inp)
 feature_vecs.append(fv.detach().cpu().numpy())
 image_ids.append(image_id[0])
 feature_vecs = np.squeeze(np.array(feature_vecs))
print(feature_vecs.shape)
 agglo = AgglomerativeClustering(n_clusters=N_CLUSTERS)
 clusters = agglo.fit_predict(feature_vecs)
 id_to_cluster = {k : v for k,v in zip(image_ids, clusters)}
del rn_model
del inp
del fv
torch.cuda.empty_cache()
return id_to_cluster<groupby>
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )
id_to_cluster = cluster_test_data()<feature_engineering>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
test_df['cluster_id'] = [id_to_cluster[iid] for iid in test_df['image_id'].values]<predict_on_test>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
def get_pseudo_labels(model, data_loader, combine_with_train=False):
model.eval()
data = []
for images, image_ids in data_loader:
predictions = []
for img in images[0]:
inp = [torch.tensor(img ).cuda().permute(2,0,1)]
features, pred, loss = model(inp)
 predictions.extend(pred)
 boxes, scores, labels = run_wbf(predictions, image_size=image_size)
 boxes =(boxes ).astype(np.int32 ).clip(min=0, max=int(image_size-1))
image_id = image_ids[0]
for box in boxes:
x0, y0, x1, y1 = box
w = x1 - x0
h = y1 - y0
data.append([image_id, 1024, 1024, 'test', x0, y0, w, h, x1, y1, image_id, join(test_path, image_id + '.jpg'), id_to_cluster[image_id]])
 s1_df = pd.DataFrame(data, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h', 'x1', 'y1', 'image_id_orig', 'image_path', 'cluster_id'])
 if combine_with_train:
s1_df = pd.concat([s1_df, train_df])
 return s1_df<load_pretrained>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
def train_round(round_num, model):
for cluster_id in range(N_CLUSTERS):
if round_num == 0:
model.load_state_dict(torch.load(weights_path))
else:
model.load_state_dict(torch.load('round_%d_cluster_%d.pth' %(round_num-1, cluster_id)))
 optimizer = torch.optim.SGD(model.parameters() , lr=0.001, momentum=0.9, weight_decay=0.0001)
 cluster_df = test_df[test_df['cluster_id'] == cluster_id]
cluster_dataset = WheatTestDataset(cluster_df, test_path)
 cluster_data_loader = DataLoader(cluster_dataset, batch_size=1, shuffle=False, num_workers=4, drop_last=False, collate_fn=collate_fn)
 cluster_df = get_pseudo_labels(model, cluster_data_loader)
 train_dataset = WheatTrainDataset(cluster_df, get_train_transforms())
 train_data_loader = DataLoader(train_dataset, batch_size=8, num_workers=4, drop_last=False, collate_fn=collate_fn)
 for epoch in range(PL_EPOCHS):
print('Round %d | Cluster %d | Epoch %d' %(round_num, cluster_id, epoch))
model.train()
for step,(images, targets, image_ids)in enumerate(train_data_loader):
images = [img.cuda() for img in images]
targets = [{k: v.cuda() for k, v in l.items() } for l in targets]
features, detections, losses = model(images, targets)
 optimizer.zero_grad()
loss_FS = sum(losses.values())
 loss_FS.backward()
optimizer.step()
torch.save(model.state_dict() , 'round_%d_cluster_%d.pth' %(round_num, cluster_id))<normalization>
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params,
n_estimators=1000 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
for round_num in range(PL_ROUNDS):
train_round(round_num, model )<define_variables>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<split>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
detection_threshold = 0.5
image_size = 1024
results = []
for cluster_id in range(N_CLUSTERS):
model.load_state_dict(torch.load('round_%d_cluster_%d.pth' %(round_num, cluster_id)))
 model.eval()
test_df_cluster = test_df[test_df['cluster_id'] == cluster_id]
test_dataset = WheatTestDataset(test_df_cluster, test_path)
 test_data_loader = DataLoader(test_dataset, batch_size=1, num_workers=4, drop_last=False, collate_fn=collate_fn)
 for images, image_ids in test_data_loader:
image_id = image_ids[0]
predictions = []
for img in images[0]:
inp = [torch.tensor(img ).cuda().permute(2,0,1)]
features, pred, loss = model(inp)
 predictions.extend(pred)
 boxes, scores, labels = run_wbf(predictions, image_size=image_size)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=int(image_size-1))
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<prepare_output>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
sub_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 sub_df.head()<save_to_csv>
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
full_data.to_csv('full_data.csv', index=False )
sub_df.to_csv('submission.csv', index=False )<install_modules>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<set_options>
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
sub_full_data.to_csv('sub_full_data.csv', index = False )
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<load_from_csv>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<define_search_model>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<init_hyperparams>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(**params,
n_estimators=1500, 
)
 print('XGB' + "-" * 50)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 print('LGB' + "-" * 50)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds1[val_] = xg.predict(val_x)
 oof_preds[oof_preds < 0] = 0
oof_preds1[oof_preds1 < 0] = 0
both_oof[val_] = oof_preds[val_] * 0.6 + oof_preds1[val_] * 0.4
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
print("LGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5)
 print("XGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1)**.5)
 print("Combine ", mean_squared_error(np.log1p(trn_user_target['target']), both_oof)**.5 )
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint1.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<init_hyperparams>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('LGB_XGB.csv', index=True )