Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<init_hyperparams>
warnings.simplefilter('error', SettingWithCopyWarning)
 warnings.simplefilter("ignore")
 gc.enable()
sns.set()
%matplotlib inline
class ConfigRNN:
model = 'rnn'
n_models = len(MODELS)
 max_len = 150
max_len_val = 150
sent_embed_dim = 16
char_embed_dim = 8
ft_lstm_dim = 16
lstm_dim = 64
use_msd = True
loss_config = {
"smoothing": True,
"neighbour_smoothing": False,
"eps": 0.1,
"use_dist_loss": False,
"dist_loss_weight": 1,
}
use_len_sampler = False
batch_size = 128
batch_size_val = 512
epochs = 10
swa_first_epoch = 5
lr = 5e-3
warmup_prop = 0.
 remove_neutral = False
pl_confidence = 0.35<define_variables>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 train.shape, test.shape
pred_oofs = []
pred_tests = []<categorify>
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'} )
for idx, comb in enumerate(combs):
print('
print(f' -> Combination {idx + 1}/{len(combs)} :
{" / ".join(list(comb)) } ')
 print('
")
 used = [model_names.index(c)for c in comb]
used_preds = {}
for key in preds.keys() :
used_preds[key] = np.array([preds[key][i][:, used] for i in range(len(preds[key])) ])
 for config in configs:
print(f' -> Training {config.model.upper() }')
 config.n_models = len(used)
 pred_oof, pred_test = k_fold(config, df_train, df_test, np.array(X_train), np.array(X_test), used_preds, len_voc,
k=K, seed=SEED, model_name='wavenet_0_0')
 pred_oofs.append(pred_oof)
 pred_tests.append(pred_test)
 print('
' )<categorify>
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str )
def string_from_preds_char_level(dataset, preds, test=False, remove_neutral=False, uncensored=False, cleaned=False):
selected_texts = []
n_models = len(preds)
 for idx in range(len(dataset)) :
data = dataset[idx]
if test:
start_probas = np.mean([preds[i][0][idx] for i in range(n_models)], 0)
 end_probas = np.mean([preds[i][1][idx] for i in range(n_models)], 0)
 else:
start_probas = preds[idx][0]
end_probas = preds[idx][1]
start_idx = np.argmax(start_probas)
 end_idx = np.argmax(end_probas)
 if end_idx < start_idx:
selected_text = data["text"]
elif remove_neutral and data["sentiment"] == "neutral":
selected_text = data["text"]
else:
selected_text = data["text"][start_idx: end_idx]
selected_texts.append(selected_text.strip())
 return selected_texts<create_dataframe>
train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False)
 for df in [train, test]:
df["visitId"] = df["visitId"].apply(lambda x: x.split('.', 1)[0] ).astype(str)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId" )
config = ConfigWav()
config.n_models = len(preds['oof_start'][0][0])
 test_dataset = TweetCharDataset(
 df_test, X_test, preds['test_start'], preds['test_end'],
max_len=config.max_len_val, train=False, n_models=config.n_models, 
)
 dataset = TweetCharDataset(
 df_train, X_train, preds['test_start'], preds['test_end'],
max_len=config.max_len_val, train=False, n_models=config.n_models,
)<prepare_output>
for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 for df in [train_new, test_new]:
df["Sessions"] = df["Sessions"].fillna(0)
 df["Avg.Session Duration"] = df["Avg.Session Duration"].fillna(0)
 df["Bounce Rate"] = df["Bounce Rate"].fillna(0)
 df["Revenue"] = df["Revenue"].fillna(0)
 df["Transactions"] = df["Transactions"].fillna(0)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].fillna(0)
 df['trafficSource.adContent'].fillna('N/A', inplace=True)
 df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)
 df['trafficSource.referralPath'].fillna('N/A', inplace=True)
 df['trafficSource.keyword'].fillna('N/A', inplace=True)
 df['totals.bounces'].fillna(0.0, inplace=True)
 df['totals.newVisits'].fillna(0.0, inplace=True)
 df['totals.pageviews'].fillna(0.0, inplace=True )
pred_oof =(np.array(pred_oofs[0])+ np.array(pred_oofs[1])+ np.array(pred_oofs[2])) / 3
pred_test = pred_tests[0] + pred_tests[1] + pred_tests[2]<compute_train_metric>
del train
del test
train = train_new
test = test_new
del train_new
del test_new
gc.collect()
selected_texts_oof = string_from_preds_char_level(dataset, pred_oof, test=False, remove_neutral=False )<compute_test_metric>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
scores = [jaccard(pred, truth)for(pred, truth)in zip(selected_texts_oof, df_train['selected_text'])]
score = np.mean(scores)
 print(f'Local CV score is {score:.4f}' )<compute_train_metric>
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
selected_texts = string_from_preds_char_level(test_dataset, pred_test, test=True, remove_neutral=False )<save_to_csv>
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )
sub['selected_text'] = selected_texts
sub.to_csv('submission.csv', index=False)
 sub.head()<define_variables>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
tqdm.pandas()
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train.head()<count_missing_values>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
train.isnull().sum()<count_missing_values>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
train.isnull().sum()<count_missing_values>
warnings.simplefilter(action='ignore', category=FutureWarning)
 folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params,
n_estimators=1000 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
train = train.dropna(axis = 0)
 train.isnull().sum()<string_transform>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
def process(text, selected_text, ib_space):
added_extra_space = False
splitted = text.split(selected_text)
 if splitted[0][-1] == ' ':
added_extra_space = True
splitted = text.split(" " + selected_text)
 sub = len(splitted[0])- len(" ".join(splitted[0].split()))
if sub == 1 and text[0] == ' ':
splitted = text.split(selected_text)
 add_space = True if splitted[0] and splitted[0][-1] == ' ' else False
add_space = False
if splitted[0] and splitted[0][-1] == ' ':
add_space = True
if add_space == False:
start = text.find(selected_text)
 else:
start = text.find(selected_text)- 1
elif sub == 2:
start = text.find(selected_text)
 else:
start = text.find(selected_text)
 splitted = text.split(selected_text)
 add_space = False
if splitted[0] and splitted[0][-1] == ' ':
add_space = True
text_pr = " ".join(splitted[0].split())
 if add_space:
text_pr += " "
text_pr = text_pr + selected_text
if len(splitted)> 1:
text_pr = text_pr + splitted[1]
if sub > 1 :
if text[0] == ' ':
end = start + len(selected_text)- 1 + ib_space
else:
end = start + len(selected_text)
 else:
end = start + len(selected_text)
 new_st = text_pr[start : end]
return new_st
def process_selected_text(text, selected_text):
splitted = text.split(selected_text)
 add_space = True if splitted[0] and splitted[0][-1] == ' ' else False
sub = len(splitted[0])- len(" ".join(splitted[0].split()))
in_between_space = len(selected_text)- len(" ".join(selected_text.split()))
new_selected_text = selected_text
if sub > 0 and text.strip() != selected_text.strip() and add_space == False and text.find(selected_text)!= 0:
if in_between_space == 0:
new_selected_text = process(text, selected_text, in_between_space)
 return new_selected_text<feature_engineering>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
train['new_selected_text'] = train.selected_text
train = train[train.textID != '12f21c8f19']
train['new_selected_text'] = train.progress_apply(lambda x: process_selected_text(x.text, x.selected_text), axis=1 )<save_to_csv>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
train.to_csv('new_train.csv',index = False )<load_from_csv>
%%time
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 trn_all_predictions.to_csv('trn_all_predictions.csv', index=False)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
train = pd.read_csv('/kaggle/working/new_train.csv')
 train.head(5 )<define_variables>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
%%time
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_all_predictions.to_csv('sub_all_predictions.csv',index = False)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
n_splits = 5<load_pretrained>
warnings.simplefilter(action='ignore', category=FutureWarning)
 folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(**params,
n_estimators=1500, 
)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5
preds_start1 = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end1 = np.zeros(( input_ids_t.shape[0],MAX_LEN))
for i in range(5):
K.clear_session()
model, padded_model = build_model()
print('load model...')
 weight_fn = '/kaggle/input/with-processing/v0-roberta-%i.h5'%(i)
 load_weights(model, weight_fn)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 preds_start1 += preds[0]/n_splits
preds_end1 += preds[1]/n_splits<save_to_csv>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('final_sub.csv', index=True )