Kaggle competition: Google Analytics Customer Revenue Prediction
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start1[k,])
 b = np.argmax(preds_end1[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st)
 test['selected_text1'] = all
pd.set_option('max_colwidth', 60)
 test.sample(25 )<categorify>
gc.enable()
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
def reverse_preprocessing(text):
text = text.replace(".... ", ".... ")
 text = text.replace("...", "...")
 text = text.replace(".. ", ".. ")
 text = text.replace("! ! ! !", "!!!!")
 text = text.replace("! ! !", "!!!")
 text = text.replace("! !", "!!")
 text = text.replace("? ? ? ?", "????")
 text = text.replace("? ? ?", "???")
 text = text.replace("? ?", "??")
 return text
def find_text_idx(text, selected_text):
text_len = len(text)
 for start_idx in range(text_len):
if text[start_idx] == selected_text[0]:
for end_idx in range(start_idx+1, text_len+1):
contained_text = "".join(text[start_idx: end_idx].split())
 if contained_text == "".join(selected_text.split()):
return start_idx, end_idx
return None, None
def calculate_spaces(text, selected_text):
selected_text = " ".join(selected_text.split())
 start_idx, end_idx = find_text_idx(text, selected_text)
 if start_idx is None:
start_idx = 0
print("----------------- error no start idx find ------------------")
 print("text:", text, "prediction:", selected_text)
 print("----------------- error no start idx find ------------------")
 if end_idx is None:
end_idx = len(text)
 print("----------------- error no end idx find ------------------")
 print("text:", text, "prediction:", selected_text)
 print("----------------- error no end idx find ------------------")
 x = text[:start_idx]
try:
if x[-1] == " ":
x = x[:-1]
except:
pass
l1 = len(x)
 l2 = len(" ".join(x.split()))
return l1 - l2, start_idx, end_idx
def pp_v2(text, predicted):
text = str(text ).lower()
predicted = predicted.lower()
predicted = predicted.strip()
if len(predicted)== 0:
return predicted
predicted = reverse_preprocessing(str(predicted))
spaces, index_start, index_end = calculate_spaces(text, predicted)
 if spaces == 1:
if len(text[max(0, index_start-1): index_end+1])<= 0 or text[max(0, index_start-1): index_end+1][-1] != ".":
return text[max(0, index_start - 1): index_end]
else:
return text[max(0, index_start-1): index_end+1]
elif spaces == 2:
return text[max(0, index_start-2): index_end]
elif spaces == 3:
return text[max(0, index_start-3): index_end-1]
elif spaces == 4:
return text[max(0, index_start-4): index_end-2]
else:
return predicted<compute_test_metric>
train_df = pd.read_csv('.. /input/data-flattened/train-flattened/train-flattened.csv', dtype={'fullVisitorId': 'str'})
 test_df = pd.read_csv('.. /input/data-flattened/test-flattened/test-flattened.csv', dtype={'fullVisitorId': 'str'} )
test["selected_text"] = test.apply(lambda x: pp_v2(x.text, x.selected_text1), axis=1)
 test.sample(10 )<save_to_csv>
const_cols = [c for c in train_df.columns if train_df[c].nunique(dropna=False)== 1]
train_df.drop(columns=const_cols + ['trafficSource.campaignCode'], inplace=True)
 test_df.drop(columns=const_cols, inplace=True )
test[['textID','selected_text']].to_csv('submission.csv',index = False)
<import_modules>
def browser_mapping(x):
browsers = ['chrome', 'safari', 'firefox', 'internet explorer', 'edge', 'opera', 'coc coc', 'maxthon', 'iron']
mobile_browsers = ['android', 'samsung', 'mini', 'iphone', 'in-app', 'playstation', 'mozilla', 'chrome',
'blackberry', 'nokia', 'browser', 'amazon', 'lunascape', 'netscape', 'konqueror', 'puffin']
if x in browsers:
return x
elif '(not set)' in x:
return x
elif sum([(k in x)for k in mobile_browsers]):
return 'mobile browser'
else:
return 'others'
def adcontents_mapping(x):
if 'google' in x:
return 'google'
elif('placement' in x)or('placememnt' in x):
return 'placement'
elif('(not set)' in x)or('nan' in x):
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
sources = ['google', 'youtube', '(not set)', 'nan', 'yahoo', 'facebook', 'reddit', 'bing', 'quora', 'outlook', 'linkedin',
'pinterest', 'ask', 'siliconvalley', 'lunametrics', 'amazon', 'mysearch', 'qiita', 'messenger', 'twitter',
't.co', 'vk.com', 'search', 'edu', 'mail', 'ad', 'golang', 'direct', 'dealspotr', 'sashihara', 'phandroid',
'baidu', 'mdn', 'duckduckgo', 'seroundtable', 'metrics', 'sogou', 'businessinsider', 'github', 'gophergala',
'yandex', 'msn', 'dfa', 'feedly', 'arstechnica', 'squishable', 'flipboard', 't-online.de', 'sm.cn', 'wow',
'partners']
for s in sources:
if s in x:
return s
return 'others'
def custom(df):
print('custom.. ')
 df['source.country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']
df['campaign.medium'] = df['trafficSource.campaign'] + '_' + df['trafficSource.medium']
df['browser.category'] = df['device.browser'] + '_' + df['device.deviceCategory']
df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']
df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']
df['content.source'] = df['trafficSource.adContent'] + "_" + df['source.country']
df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[i + "_" + j] = df[i] + "_" + df[j]
return df
for df in [train_df, test_df]:
df['device.browser'] = df['device.browser'].map(lambda x: browser_mapping(str(x ).lower()))
df['trafficSource.adContent'] = df['trafficSource.adContent'].map(lambda x: adcontents_mapping(str(x ).lower()))
df['trafficSource.source'] = df['trafficSource.source'].map(lambda x: source_mapping(str(x ).lower()))
train_df = custom(train_df)
 test_df = custom(test_df )
import numpy as np
import pandas as pd
import os
import tokenizers
import string
import torch
import transformers
import torch.nn as nn
from torch.nn import functional as F
from tqdm import tqdm
import re<define_variables>
for df in [train_df, test_df]:
df['visitStartTime'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['visitStartTime'].dt.dayofweek
df['sess_date_hours'] = df['visitStartTime'].dt.hour
df['totals.hits/views'] = df['totals.hits'] /(df['totals.pageviews'] + 1)
 df.sort_values(['fullVisitorId', 'visitStartTime'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['visitStartTime'] - df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['visitStartTime'] - df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
MAX_LEN = 92
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 8
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)<define_search_model>
num_cols = ['visitNumber', 'totals.hits', 'totals.pageviews', 'totals.hits/views', 'next_session_1', 'next_session_2']
excluded_cols = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime']
cat_cols = [col for col in train_df.columns
if(col not in excluded_cols)and(col not in num_cols)]
all_cols = cat_cols + num_cols
for col in cat_cols:
train_df[col], indexer = pd.factorize(train_df[col].astype(str))
test_df[col] = indexer.get_indexer(test_df[col].astype(str))
for col in num_cols:
train_df[col] = train_df[col].astype(float)
 test_df[col] = test_df[col].astype(float)
 train_df["totals.transactionRevenue"].fillna(0, inplace=True )
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<define_variables>
def get_folds(df=None, n_splits=5, seed=42):

 unique_vis = np.array(sorted(df['fullVisitorId'].unique()))
folds = GroupKFold(n_splits=n_splits)
 fold_ids = []
ids = np.arange(df.shape[0])
 np.random.seed(seed)
 for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
fold_ids.append(
 [
ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],
ids[df['fullVisitorId'].isin(unique_vis[val_vis])]
] 
)
 return fold_ids
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + str(tweet ).lstrip()
selected_text = " " + str(selected_text ).lstrip()
len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 try:
targets_start = target_idx[0]
targets_end = target_idx[-1]
except:
print(tweet)
 targets_start = 0
targets_end = -1
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<define_variables>
params = {'learning_rate': 0.01,
'objective': 'regression',
'metric': 'rmse',
'num_leaves': 49,
'verbose': 1,
'bagging_fraction': 0.94,
'feature_fraction': 0.67,
'random_state': 42,
'max_depth': 14,
'random_seed': 42,
'bagging_frequency': 5,
'lambda_l2': 0.2,
'lambda_l1': 0.55,
'min_child_samples': 130
}
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_start = 0
idx_end = 91
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 4:
filtered_output = original_tweet
if sentiment_val != "neutral" and verbose == True:
if filtered_output.strip().lower() != target_string.strip().lower() :
print("********************************")
 print(f"Output= {filtered_output.strip() }")
 print(f"Target= {target_string.strip() }")
 print(f"Tweet= {original_tweet.strip() }")
 print("********************************")
 jac = 0
return jac, filtered_output<categorify>
train_y = train_df["totals.transactionRevenue"]
folds = get_folds(train_df, n_splits=5)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train_df[all_cols].iloc[trn_], train_y.iloc[trn_]
val_x, val_y = train_df[all_cols].iloc[val_], train_y.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=2000)
 reg.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ], early_stopping_rounds=100,
verbose=100, eval_metric='rmse')
 imp_df = pd.DataFrame()
imp_df['feature'] = all_cols
imp_df['gain_reg'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
lgb_preds = reg.predict(test_df[all_cols], num_iteration=reg.best_iteration_)
 lgb_preds[lgb_preds < 0] = 0
sub_preds += np.expm1(lgb_preds)/ len(folds)
 print("LGBM session-level error: ", mean_squared_error(np.log1p(train_y.values), oof_reg_preds)**.5)
 oof_pred_df = pd.DataFrame({"fullVisitorId": train_df["fullVisitorId"].values})
 oof_pred_df["transactionRevenue"] = train_df["totals.transactionRevenue"].values
oof_pred_df["PredictedRevenue"] = np.expm1(oof_reg_preds)
 oof_pred_df = oof_pred_df.groupby("fullVisitorId")["transactionRevenue", "PredictedRevenue"].sum().reset_index()
oof_err = np.sqrt(mean_squared_error(np.log1p(oof_pred_df["transactionRevenue"].values),
np.log1p(oof_pred_df["PredictedRevenue"].values)))
 print("LGBM user-level error: ", oof_err )
def simple_fix(a):
a = str(a)
 a = a.lower()
a = a.replace("n`****", "n`t it")
 a = a.replace("`", "'")
 a = a.replace('"', "'")
 a = a.replace("Â", "_")
 a = a.replace("ï¿½", "_'_")
 return a<feature_engineering>
train_df['predictions'] = np.expm1(oof_reg_preds)
 test_df['predictions'] = sub_preds
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "original_text"] = df_test.text.values
df_test['text'] = df_test.text.map(simple_fix)
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
def find_mode(x):
mode, mode_cnt = 0, 0
d = dict()
for val in x.values:
current_cnt = d.get(val, 0)+ 1
d[val] = current_cnt
if current_cnt > mode_cnt:
mode = val
mode_cnt = current_cnt
return mode
def parse_predictions(df):
top_N = 20
res = dict()
res['pred_first'] = df.predictions.values[0]
res['pred_last'] = df.predictions.values[-1]
for i, pred in enumerate(df.predictions.sort_values(ascending=False ).head(top_N)) :
res['pred_' + str(i)] = pred
return res
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True<load_pretrained>
%%time
trn_data = train_df[all_cols + ['fullVisitorId']].groupby('fullVisitorId')\ 
.agg({c: np.mean if c in num_cols else find_mode for c in all_cols})
 trn_pred_list = train_df[['fullVisitorId', 'predictions']].groupby('fullVisitorId' ).apply(parse_predictions)
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index )
model0 = TweetModel(conf=model_config)
 model0.to(device)
 model0.load_state_dict(torch.load(".. /input/strong-v2/model_f0_s696.bin"))
model0.eval()
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/strong-v2/model_f1_s696.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(".. /input/strong-v2/model_f2_s696.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(".. /input/strong-v2/model_f3_s696.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(".. /input/strong-v2/model_f4_s696.bin"))
model4.eval()
1+1<define_variables>
session_pred_cols = trn_all_predictions.columns[:-2]
trn_all_predictions['t_log_mean'] = np.log1p(trn_all_predictions[session_pred_cols].mean(axis=1))
trn_all_predictions['t_log_median'] = np.log1p(trn_all_predictions[session_pred_cols].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[session_pred_cols] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[session_pred_cols].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[session_pred_cols].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
final_output = []<create_dataframe>
%%time
sub_data = test_df[all_cols + ['fullVisitorId']].groupby('fullVisitorId')\ 
.agg({c: np.mean if c in num_cols else find_mode for c in all_cols})
 sub_pred_list = test_df[['fullVisitorId', 'predictions']].groupby('fullVisitorId' ).apply(parse_predictions)
 sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in session_pred_cols:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_log_mean'] = np.log1p(sub_all_predictions[session_pred_cols].mean(axis=1))
sub_all_predictions['t_log_median'] = np.log1p(sub_all_predictions[session_pred_cols].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[session_pred_cols] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[session_pred_cols].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[session_pred_cols].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start0, outputs_end0 = model0(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(1.1*outputs_start0 + outputs_start1 + 1.3* outputs_start2 + 0.9*outputs_start3 + outputs_start4)/ 5
outputs_end =(1.1*outputs_end0 + outputs_end1 + 1.3* outputs_end2 + 0.9*outputs_end3 + outputs_end4)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<feature_engineering>
target = train_df[['fullVisitorId', 'totals.transactionRevenue']].groupby('fullVisitorId' ).sum()
df_test.loc[:, 'raw_output'] = final_output
df_test.head(5 )<load_pretrained>
params = {'learning_rate': 0.01,
'objective': 'regression',
'metric': 'rmse',
'num_leaves': 31,
'verbose': 1,
'bagging_fraction': 0.93,
'feature_fraction': 0.57,
'random_state': 42,
'max_depth': 14,
'random_seed': 42,
'bagging_frequency': 5,
'lambda_l2': 0.62,
'lambda_l1': 0.07,
'min_child_samples': 179
}
model5 = TweetModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(".. /input/20200510-f3-s567/model_f3_s657.bin"))
model5.eval()
1+1<define_variables>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_reg_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], target['totals.transactionRevenue'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], target['totals.transactionRevenue'].iloc[val_]
reg = lgb.LGBMRegressor(**params,n_estimators=2000)
 reg.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ], early_stopping_rounds=100,
verbose=100, eval_metric='rmse')
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
lgb_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 lgb_preds[lgb_preds < 0] = 0
sub_preds += np.expm1(lgb_preds)/ len(folds)
 print("LGBM Result: ", mean_squared_error(np.log1p(target['totals.transactionRevenue']), oof_reg_preds)**.5 )
<string_transform><EOS> 
sub_full_data['PredictedLogRevenue'] = np.log1p(sub_preds)
 sub_full_data[['PredictedLogRevenue']].to_csv('submission.csv', index=True )