Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<feature_engineering>
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
def make_mapping(data_df):
data_df['device.browser'] = data_df['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 data_df['trafficSource.adContent'] = data_df['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 data_df['trafficSource.source'] = data_df['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 return data_df
def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
def process(data):
return custom(process_device(make_mapping(data)))
html =
def remove_html(text):
re_html = re.compile(r'<.*?>')
 return re_html.sub('', text)
 print(remove_html(html))
train['text'] = train['text'].apply(lambda x : remove_html(x))
test['text'] = test['text'].apply(lambda x : remove_html(x))<drop_column>
target_key = 'totals.transactionRevenue'
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text ).strip()
print(remove_emoji("Difficult kernel 😔😔"))
train['text'] = train['text'].apply(lambda x: remove_emoji(x))
test['text'] = test['text'].apply(lambda x: remove_emoji(x))<feature_engineering>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape
punct = 'Cristiano.is
def remove_puncts(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table ).strip()
print(remove_puncts(punct))
train['text'] = train['text'].apply(lambda x: remove_puncts(x))
test['text'] = test['text'].apply(lambda x: remove_puncts(x))<count_unique_values>
train['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0 )
num_classes = train.target.nunique()
num_classes<categorify>
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'} )
n_train = train.shape[0]
input_ids = np.ones(( n_train, MAX_LEN), dtype='int32')
 mask = np.zeros(( n_train, MAX_LEN), dtype='int32')
 for k in range(train.shape[0]):
text = train.loc[k, 'text']
output = roberta_tokenizer.encode_plus(text, max_length=MAX_LEN, pad_to_max_length=True)
 input_ids[k] = output['input_ids']
mask[k] = output["attention_mask"]<categorify>
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str)
 train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False )
n_test = test.shape[0]
input_ids_t = np.ones(( n_test, MAX_LEN), dtype='int32')
 mask_t = np.zeros(( n_test, MAX_LEN), dtype='int32')
 for k in range(test.shape[0]):
text = test.loc[k, 'text']
output = roberta_tokenizer.encode_plus(text, max_length=MAX_LEN, pad_to_max_length=True)
 input_ids_t[k] = output['input_ids']
mask_t[k] = output["attention_mask"]<prepare_x_and_y>
for df in [train, test, train_exdata, test_exdata]:
df["visitId"] = df["visitId"].astype(float ).astype(int)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId")
 for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 df["Revenue"] = np.log1p(df["Revenue"])
 df['is_high_hits'] = np.logical_or(df["totals.hits"]>4,df["totals.pageviews"]>4 ).astype(np.int32)
 df['views/hits']=df["totals.pageviews"]/df["totals.hits"].dropna(0 )
def create_dataset() :
xtrain = [input_ids, mask]
xtest = [input_ids_t, mask_t]
ytrain = tf.keras.utils.to_categorical(train['target'].values.reshape(-1, 1))
return xtrain, ytrain, xtest<prepare_x_and_y>
%%time
train = process(train_new)
 test = process(test_new)
 del train_new, test_new
xtrain, ytrain, xtest = create_dataset()
print("X train : {0}".format(len(xtrain[0])))
 print("Y train : {0}".format(len(ytrain)))
 print("X test : {0}".format(len(xtest[0])) )<choose_model_class>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
def build_model() :
roberta = transformers.TFRobertaForSequenceClassification.from_pretrained('roberta-base')
 optim = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE, clipnorm=2.0)
 loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 roberta.compile(loss='binary_crossentropy', optimizer=optim, metrics=['accuracy'])
 return roberta<prepare_x_and_y>
def freq_encoding(column):
counter = column.value_counts()
counter /= counter.sum()
return counter
Folds = 5
kfold = KFold(n_splits=Folds)
 predictions = list()
for i,(train_idx, test_idx)in enumerate(kfold.split(xtrain[0])) :
xtrain_fold = [xtrain[i][train_idx] for i in range(len(xtrain)) ]
xvalid_fold = [xtrain[i][test_idx] for i in range(len(xtrain)) ]
ytrain_fold = ytrain[train_idx]
yvalid_fold = ytrain[test_idx]
positive = train.iloc[train_idx, :].target.value_counts() [0]
negative = train.iloc[train_idx, :].target.value_counts() [1]
pos_weight = positive /(positive + negative)
 neg_weight = negative /(positive + negative)
 class_weight = [{0:pos_weight, 1:neg_weight}, {0:neg_weight, 1:pos_weight}]
tf.keras.backend.clear_session()
roberta = build_model()
roberta.fit(xtrain_fold, ytrain_fold,
batch_size=TRAIN_BATCH_SIZE,
epochs=EPOCHS,
class_weight=class_weight,
validation_data=(xvalid_fold, yvalid_fold))
val_preds = roberta.predict(xvalid_fold, batch_size=VALID_BATCH_SIZE, verbose=1)
 val_preds = np.argmax(val_preds, axis=1 ).flatten()
print(metrics.accuracy_score(train.iloc[test_idx, :].target.values, val_preds))
preds = roberta.predict(xtest, batch_size=TRAIN_BATCH_SIZE, verbose=1)
 predictions.append(preds)
<save_to_csv>
def mean_encoding(column, target, alpha = 0.01):
global_mean = target.mean()
gb = target.groupby(column)
 mean = gb.mean()
count = gb.count()
mean_adj =(mean * count + global_mean * alpha)/(count + alpha)
 return mean_adj
sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 predictions = np.average(predictions, axis=0)
 predictions = np.argmax(predictions, axis=1 ).flatten()
sample_submission['target'] = predictions
sample_submission['target'].value_counts()
sample_submission.to_csv('submission.csv', index=False )<import_modules>
def get_subfolds(train, fold_idx, n_subfolds):
subfolds = get_folds(df=train.loc[fold_trn_idx], n_splits=n_subfolds)
 for i in range(len(subfolds)) :
for j in range(len(subfolds[i])) :
subfolds[i][j] = fold_trn_idx[subfolds[i][j]]
return subfolds
print('Transformers version: ', transformers.__version__)
 print('Tensorflow version: ', tf.__version__ )<load_from_csv>
n_folds = 2
n_subfolds = 3
subfolds = []
folds = get_folds(df=train, n_splits=n_folds)
 for fold_trn_idx, fold_val_idx in tqdm_notebook(folds):
subfolds_cur = get_subfolds(train, fold_val_idx, n_subfolds)
 subfolds.append(subfolds_cur)
randomseed=42
bertmodel='bert-large-uncased'
data_dir = '/kaggle/input/nlp-getting-started/'
train_df = pd.read_csv(data_dir+'train.csv')
 test_df = pd.read_csv(data_dir+'test.csv')
 train_df = train_df.sample(n=len(train_df), random_state=randomseed)
 sample_submission = pd.read_csv(data_dir+'sample_submission.csv')
<prepare_x_and_y>
x_train = train_df['text']
y_train = train_df['target']
x_test = test_df['text']<load_pretrained>
tokenizer = transformers.BertTokenizer.from_pretrained(bertmodel, do_lower_case=True )<categorify>
for f in tqdm_notebook(categorical_features):
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
def encode_tweet(df):
input_ids = []
for x in df:
encoded_x = tokenizer.encode(x,
add_special_tokens = True)
 input_ids.append(encoded_x)
 return input_ids<categorify>
y_reg = train[target_key]
test_input = encode_tweet(x_test)
<feature_engineering>
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['nb_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 y_reg = train['target']
del train['target']
train_input = encode_tweet(x_train )<categorify>
del train['vis_date']
MAX_LEN = 84
def pad_tweets(df):
df = pad_sequences(df, maxlen=MAX_LEN, dtype="long",
value=0, truncating="post", padding="post")
 return df<categorify>
train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(tqdm_notebook(folds)) :
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=100,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=10 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
train_input = pad_tweets(train_input)
 test_input = pad_tweets(test_input )<categorify>
mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
def get_att_mask(df):
attention_masks = []
for tweet in df:
att_mask = [int(token_id > 0)for token_id in tweet]
attention_masks.append(att_mask)
 return attention_masks<categorify>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_std'] = np.log1p(trn_all_predictions[trn_feats].std(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
train_att = get_att_mask(train_input)
 test_att = get_att_mask(test_input )<split>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
del test
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_std'] = np.log1p(sub_all_predictions[trn_feats].std(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
tr_input, val_input, tr_label, val_label = train_test_split(train_input, y_train,
random_state=2020, test_size=0.15)
 tr_mask, val_mask, _, _ = train_test_split(train_att, y_train,
random_state=2020, test_size=0.15 )<data_type_conversions>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
del train
gc.collect()
tr_input = torch.tensor(tr_input)
 val_input = torch.tensor(val_input)
 tr_label = torch.tensor(np.array(tr_label))
val_label = torch.tensor(np.array(val_label))
tr_mask = torch.tensor(tr_mask)
 val_mask = torch.tensor(val_mask)
 te_input = torch.tensor(test_input)
 te_mask = torch.tensor(test_att )<create_dataframe>
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
BATCH_SIZE = 32
train_data = TensorDataset(tr_input, tr_mask, tr_label)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = BATCH_SIZE)
 val_data = TensorDataset(val_input, val_mask, val_label)
 val_sampler = RandomSampler(val_data)
 val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size = BATCH_SIZE )<load_pretrained>
%%time
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(tqdm_notebook(folds)) :
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=100,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
both_oof[val_] = oof_preds[val_]
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds +=(_preds / len(folds))
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5
model = BertForSequenceClassification.from_pretrained(bertmodel,
num_labels = 2,
output_attentions = False,
output_hidden_states = False )<train_model>
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5
<init_hyperparams><EOS> 
sub_full_data['PredictedLogRevenue'] = sub_preds
idx = list(set(sub_full_data.index ).intersection(set(full_data.index)))
 a = full_data.loc[idx]['Revenue']
diff = np.expm1(a[a>-10])- trn_user_target['target'].loc[idx][a>-10] / 5e5
diff[diff < 0] = 0
sub_full_data['PredictedLogRevenue'].loc[diff.index] = np.log1p(diff * 5e5)
 sub_full_data[['PredictedLogRevenue']].to_csv('new_test_leak_xgb.csv', index=True )