Kaggle competition: Google Analytics Customer Revenue Prediction
def data_preprocess(df, max_tweet_len, datasets='test', token_gap=0, debug_mode=False, offset_return=True):
try:
assert(type(df)== pd.DataFrame)
 except AssertionError:
raise AssertionError("Input dataframe is not type pandas.DataFrame!")
 dict_sentiment = {'positive': robdist_tokenize('positive')['input_ids'],
'negative': robdist_tokenize('negative')['input_ids'],
'neutral': robdist_tokenize('neutral')['input_ids']}
enc_sentiment_len = len(dict_sentiment['neutral'])
 zero_temp = robdist_tokenize(" " + str(' '.join(str(df.iloc[0]['text'] ).split())))
 df_BPE = torch.cat(( dict_sentiment[df.iloc[0]['sentiment']], zero_temp['input_ids']), dim=-1)
 df_att = torch.from_numpy(np.asarray([1]*enc_sentiment_len + [0]*token_gap + [1]*len(zero_temp['input_ids'])+ [0]*(max_tweet_len - enc_sentiment_len - token_gap - len(zero_temp['input_ids']))))
df_tok = torch.from_numpy(np.asarray([0]*(enc_sentiment_len + token_gap)+ [1]*len(zero_temp['input_ids'])+ [0]*(max_tweet_len - enc_sentiment_len - token_gap - len(zero_temp['input_ids']))))
df_textid = robdist_tokenize(str(df.iloc[0]['textID'])) ['input_ids']
df_offsets = offset_paddings(zero_temp['offsets'], max_tweet_len)
 if df_BPE.shape[0] < max_tweet_len:
df_BPE = torch.cat(( df_BPE, torch.zeros(max_tweet_len - df_BPE.shape[0], dtype=torch.long)) , -1)
 else:
df_BPE = df_BPE[:max_tweet_len]
if df_textid.shape[0] < max_tweet_len:
df_textid = torch.cat(( df_textid, torch.zeros(max_tweet_len - df_textid.shape[0], dtype=torch.long)) , -1)
 else:
df_textid = df_textid[:max_tweet_len]
if datasets=='train':
init_search = True
targetidx_1, targetidx_2, _ = sentiment_padding(' '.join(df.iloc[0]['text'].split()), ' '.join(df.iloc[0]['selected_text'].split()), max_tweet_len)
 for i in range(df_offsets.shape[0]):
if init_search and df_offsets[i, 0] <= targetidx_1[0] < df_offsets[i, 1]:
init_search = False
targetidx_1 = [i]
if df_offsets[i, 0] <= targetidx_2[0] <= df_offsets[i, 1]:
targetidx_2 = [i]
break
df_att = df_att.unsqueeze(dim=-1)
 df_tok = df_tok.unsqueeze(dim=-1)
 df_BPE = df_BPE.unsqueeze(dim=-1)
 df_textid = df_textid.unsqueeze(dim=-1)
 df_offsets = df_offsets.unsqueeze(dim=0)
 for idx in range(1, len(df)) :
tempstring = str(' '.join(str(df.iloc[idx]['text'] ).split()))
if len(tempstring)< 1:
continue
ins_ids = robdist_tokenize(" " + str(' '.join(str(df.iloc[idx]['text'] ).split())))
 text_unique_ids = robdist_tokenize(str(df.iloc[idx]['textID'])) ['input_ids']
temp_token = torch.cat(( dict_sentiment[df.iloc[idx]['sentiment']], ins_ids['input_ids']), dim=-1)
 temp_att = torch.from_numpy(np.asarray([1]*enc_sentiment_len + [0]*token_gap + [1]*len(ins_ids['input_ids'])+ [0]*(max_tweet_len - enc_sentiment_len - token_gap - len(ins_ids['input_ids']))))
temp_tok = torch.from_numpy(np.asarray([0]*(enc_sentiment_len + token_gap)+ [1]*len(ins_ids['input_ids'])+ [0]*(max_tweet_len - enc_sentiment_len - token_gap - len(ins_ids['input_ids']))))
if temp_token.shape[0] < max_tweet_len:
temp_token = torch.cat(( temp_token, torch.zeros(max_tweet_len - temp_token.shape[0], dtype=torch.long)) , dim=0)
 else:
temp_token = temp_token[:max_tweet_len]
if text_unique_ids.shape[0] < max_tweet_len:
text_unique_ids = torch.cat(( text_unique_ids, torch.zeros(max_tweet_len - text_unique_ids.shape[0], dtype=torch.long)) , dim=0)
 else:
text_unique_ids = text_unique_ids[:max_tweet_len]
df_BPE = torch.cat(( df_BPE, temp_token.unsqueeze(dim=-1)) , dim=-1)
 df_textid = torch.cat(( df_textid, text_unique_ids.unsqueeze(dim=-1)) , dim=-1)
 df_att = torch.cat(( df_att, temp_att.unsqueeze(dim=-1)) , dim=-1)
 df_tok = torch.cat(( df_tok, temp_tok.unsqueeze(dim=-1)) , dim=-1)
 df_offsets = torch.cat(( df_offsets, offset_paddings(ins_ids['offsets'], max_tweet_len ).unsqueeze(dim=0)) , dim=0)
 if datasets=='train':
init_search = True
temp_idx1, temp_idx2, _ = sentiment_padding(' '.join(str(df.iloc[idx]['text'] ).split()), ' '.join(str(df.iloc[idx]['selected_text'] ).split()), max_tweet_len)
 for i in range(df_offsets.shape[1]):
if init_search and df_offsets[idx, i, 0] <= temp_idx1[0] < df_offsets[idx, i, 1]:
init_search = False
temp_idx1 = [i]
if df_offsets[idx, i, 0] <= temp_idx2[0] <= df_offsets[idx, i, 1]:
temp_idx2 = [i]
break
targetidx_1 = targetidx_1 + temp_idx1
targetidx_2 = targetidx_2 + temp_idx2
if datasets=='test':
return {
'BPEencoded': df_BPE.T,
'textID': df_textid.T,
'att_mask': df_att.T,
'tok_typeids': df_tok.T,
'offsets': df_offsets
}
elif datasets=='train':
return {
'BPEencoded': df_BPE.T,
'textID': df_textid.T,
'target_idx': torch.from_numpy(np.stack(( np.asarray(targetidx_1), np.asarray(targetidx_2)) , axis=0)).T,
'att_mask': df_att.T,
'tok_typeids': df_tok.T,
'offsets': df_offsets
}
else:
return -1<define_variables>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline
data_path = '/kaggle/input/tweet-sentiment-extraction'
config_path = '/kaggle/input/submission'
prep_out_path = '/kaggle/working'
temporary_tweet = '/kaggle/input/roberta-openai'<concatenate>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape
def stackup_ids(input_ids, mask_ids, tokentype_ids, textID_ids, offsets, bertmodel='roberta'):
try:
assert(input_ids.shape == mask_ids.shape == tokentype_ids.shape)
 assert(input_ids.shape[0] == textID_ids.shape[0])
 except AssertionError:
raise AssertionError('stackup_ids: arguments do not have matched shape!')
 token_type_modifier = tokentype_ids*int(0)if bertmodel=='roberta' else tokentype_ids
cat_out = torch.cat(( input_ids.unsqueeze(dim=-1),
mask_ids.long().unsqueeze(dim=-1),
token_type_modifier.long().unsqueeze(dim=-1),
textID_ids.unsqueeze(dim=-1),
offsets.long()), dim=-1)
 return cat_out<load_pretrained>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
MAX_ORIG_TWEET_LEN = 192
BATCH_SIZE = 32
dev = 'cuda' if torch.cuda.is_available() else 'cpu'
fold_run = 1/0.05
try:
train_preprocess, test_preprocess = pickle.load(open(os.path.join(temporary_tweet, 'tweet_preprocess.p'), mode='rb'))
print("Pre-processed tweet dataset loaded!")
 except FileNotFoundError:
print("Initializing/Repeating preprocess of data!")
 train_preprocess = data_preprocess(train_dataframe, MAX_ORIG_TWEET_LEN, datasets='train')
 test_preprocess = data_preprocess(test_dataframe, MAX_ORIG_TWEET_LEN, datasets='test')
 pickle.dump(( train_preprocess, test_preprocess), open(os.path.join(prep_out_path, 'tweet_preprocess.p'), 'wb'))
print('Preprocessing completed and saved the data!')
 val_ratio = float(1/fold_run)
 val_slice_idx = int(( 1-val_ratio)*train_preprocess['BPEencoded'].shape[0])
 train_x_ids = stackup_ids(train_preprocess['BPEencoded'][0:val_slice_idx, :],
train_preprocess['att_mask'][0:val_slice_idx, :].long() ,
train_preprocess['tok_typeids'][0:val_slice_idx, :].long() ,
train_preprocess['textID'][0:val_slice_idx, :],
train_preprocess['offsets'][0:val_slice_idx, :].long())
 train_y = train_preprocess['target_idx'][0:val_slice_idx, :]
val_x_ids = stackup_ids(train_preprocess['BPEencoded'][val_slice_idx:, :],
train_preprocess['att_mask'][val_slice_idx:, :].long() ,
train_preprocess['tok_typeids'][val_slice_idx:, :].long() ,
train_preprocess['textID'][val_slice_idx:, :],
train_preprocess['offsets'][val_slice_idx:, :].long())
 val_y = train_preprocess['target_idx'][val_slice_idx:, :]
train_x_ids_dataset = TensorDataset(train_x_ids, train_y)
 train_loader = DataLoader(train_x_ids_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=1)
 val_x_ids_dataset = TensorDataset(val_x_ids, val_y)
 val_loader = DataLoader(val_x_ids_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=1 )<string_transform>
train['target'] = y_reg
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
y_reg = train['target']
del train['target']
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def char_jaccard_seq(str1, str2):
jaccscore = 0.0
a = list(''.join(str1.lower().split()))
b = list(''.join(str2.lower().split()))
minlen = len(a)if len(a)< len(b)else len(b)
 for idx in range(minlen):
if a[idx] == b[idx]:
jaccscore += 1
return(jaccscore*2)/(len(a)+ len(b))<prepare_x_and_y>
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )
stopper= 1
devv = 'cuda' if torch.cuda.is_available() else 'cpu'
for i, batch in enumerate(train_loader):
if stopper < 1:
break
batchx, batchy = batch
batchx, batchy = batchx.to(devv), batchy.to(devv)
 stopper -= 1
textidd = robdist_detokenize(batchx[0,:,3])
 print(train_dataframe.loc[train_dataframe['textID'] == textidd]['text'].item())
 print(robdist_detokenize(batchx[0,:,0]))
print(batchy[0,:])
 print('---------------------------------------------------------------------------------------------------------------')
 train1 = train_dataframe.loc[train_dataframe['textID'] == textidd]['selected_text'].item()
train2 = str(" " + ' '.join(train_dataframe.loc[train_dataframe['textID'] == textidd]['text'].item().split())) [batchx[0,:,-2:][batchy[0,0], 0]:batchx[0,:,-2:][batchy[0,1], 1]]
print(train1)
 print(train2)
 print(f'Jaccard {jaccard(train1,train2)}')
 print('---------------------------------------------------------------------------------------------------------------')
 print(batchx[0,:40,-2:].squeeze(dim=0 ).T)
 print(batchx[0,:50,:])
 del devv, textidd, train1, train2<define_variables>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
roberta_path = config_path
model_in = '/kaggle/input/submission/model_colab_24.pt'<load_pretrained>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
def weights_init(mods):
if type(mods)== nn.Linear:
torch.nn.init.normal_(mods.weight, mean= 0.0, std=0.1)
 class roberta_mlp_net(BertPreTrainedModel):
def __init__(self, robpath, net_config, maxtweetlen):
super(roberta_mlp_net, self ).__init__(net_config)
 net_config.output_hidden_states = True
self.roberta = RobertaModel.from_pretrained(pretrained_model_name_or_path=robpath, config=net_config)
 self.drop_out1 = nn.Dropout(p=0.1)
 self.laynorm = nn.LayerNorm(( 768*2,))
self.densenet = nn.Linear(768*2, 2)
 self.densenet.apply(weights_init)
 def todevice(self):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
self.to(device)
 print(f'Processing device: {device}')
 def forward(self, in_ids, att_ids, tok_ids):
_ , _, out = self.roberta(input_ids=in_ids, attention_mask=att_ids, token_type_ids=tok_ids)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out1(out)
 out = self.laynorm(out)
 out = self.densenet(out)
 return out<choose_model_class>
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
learn_rate = 80e-7
criterion_mod = nn.CrossEntropyLoss()
global_threshold_valid = np.Inf
n_epoch = 30
print_every = 100
model_config = PretrainedConfig.from_json_file(os.path.join(roberta_path, 'roberta-base-openai-detector-config.json'))
testmodel = roberta_mlp_net(robpath=roberta_path, net_config=model_config, maxtweetlen=MAX_ORIG_TWEET_LEN)
 optimizer_mod = optim.AdamW(testmodel.parameters() , lr=learn_rate)
 optimizer_mod.param_groups[0]['amsgrad'] = False
if dev == 'cuda':
testmodel.load_state_dict(torch.load(model_in)['model'])
 else:
testmodel.load_state_dict(torch.load(model_in, map_location='cpu')['model'] )<compute_train_metric>
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
def len_mismatch_penalty(pred_start, pred_end, tar_start, tar_end, coeff_mult=0.1):
_, start_cand = pred_start.squeeze(dim=-1 ).topk(1, dim=-1)
 _, end_cand = pred_end.squeeze(dim=-1 ).topk(1, dim=-1)
 start_cand = start_cand.squeeze(dim=-1)
 end_cand = end_cand.squeeze(dim=-1)
 diff_cand = abs(end_cand - start_cand)-(tar_end - tar_start)
 result = torch.matmul(diff_cand.float() , diff_cand.float().T)
 return result*coeff_mult<find_best_params>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
def train_func(fold, model, dev, optimizer, trainloader, validloader, tweetlen, criterion, epochs, glob_th, verbose_every=10, first_train=True):
valid_loss_min = np.Inf
init_train = True
early_stop = 2
prev_val_loss = np.Inf
model.todevice()
for epoch in range(1, epochs + 1):
train_loss = 0.0
valid_loss = 0.0
if 3 < epoch < 5:
optimizer.param_groups[0]['lr'] = 0.8*optimizer.param_groups[0]['lr']
elif 5 < epoch < 10:
optimizer.param_groups[0]['lr'] = 0.8*optimizer.param_groups[0]['lr']
elif epoch > 10:
optimizer.param_groups[0]['lr'] = 0.8*optimizer.param_groups[0]['lr']
model.train()
print(f'Fold {fold} : Training of epoch
for batch_idx,(trainx, trainy)in enumerate(trainloader):
model.zero_grad()
trainx, trainy = trainx.to(dev), trainy.to(dev)
 output = model(trainx[:, :, 0], trainx[:, :, 1], trainx[:, :, 2])
 loss =(criterion(output[:, :, 0], trainy[:, 0])+ criterion(output[:, :, 1], trainy[:, 1]))
loss.backward()
optimizer.step()
train_loss += loss.item()
if batch_idx % verbose_every == 0 and batch_idx != 0:
print(f'Training loss at epoch
train_loss = 0.0
if init_train and not first_train:
valid_loss_min = glob_th
init_train = False
model.eval()
print(f'Validation of epoch
for batch_idx,(valx, valy)in enumerate(validloader):
valx, valy = valx.to(dev), valy.to(dev)
 with torch.no_grad() :
val_output = model(valx[:, :, 0], valx[:, :, 1], valx[:, :, 2])
 vloss = criterion(val_output[:, :, 0], valy[:, 0])+ criterion(val_output[:, :, 1], valy[:, 1])
 valid_loss += vloss.item()
valid_loss = valid_loss/len(validloader)
 print('Validation loss : {:.6f}'.format(valid_loss))
print('Validation loss min : {:.6f}'.format(valid_loss_min))
if valid_loss < valid_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(valid_loss_min, valid_loss))
valid_loss_min = valid_loss
glob_th = valid_loss
checkpoint_model = {'architecture': save_architecture,
'learnrate': learn_rate,
'validmin': valid_loss_min,
'model': model.state_dict() }
torch.save(checkpoint_model, os.path.join(model_out))
early_stop = 2
else:
if prev_val_loss < valid_loss:
early_stop -= 1
else:
early_stop = 2
if early_stop < 0:
break
prev_val_loss = valid_loss
checkpoint_model = {'architecture': save_architecture,
'learnrate': learn_rate,
'validmin': glob_th,
'model': model.state_dict() }
torch.save(checkpoint_model, os.path.join(model_last_out))
print('Training completed!')
 return model, glob_th<train_model>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
testmodel, global_threshold_valid = train_func(fold=0,
model=testmodel,
dev=dev,
optimizer=optimizer_mod,
trainloader=train_loader,
validloader=val_loader,
tweetlen=MAX_ORIG_TWEET_LEN,
criterion=criterion_mod,
epochs=n_epoch,
glob_th=global_threshold_valid,
verbose_every=print_every,
first_train=False )<define_variables>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
<string_transform>
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
def pp(filtered_output, real_tweet):
filtered_output = ' '.join(filtered_output.split())
 if len(real_tweet.split())< 2:
filtered_output = real_tweet
else:
if len(filtered_output.split())== 1:
if filtered_output.endswith(".. "):
if real_tweet.startswith(" "):
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\.) \1{2,}', '', filtered_output)
 else:
filtered_output = re.sub(r'(\.) \1{2,}', '.', filtered_output)
 else:
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\.) \1{2,}', '.', filtered_output)
 else:
filtered_output = re.sub(r'(\.) \1{2,}', '.. ', filtered_output)
 return filtered_output
if filtered_output.endswith('!!'):
if real_tweet.startswith(" "):
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\!)\1{2,}', '', filtered_output)
 else:
filtered_output = re.sub(r'(\!)\1{2,}', '!', filtered_output)
 else:
st = real_tweet.find(filtered_output)
 fl = real_tweet.find(" ")
 if fl != -1 and fl < st:
filtered_output = re.sub(r'(\!)\1{2,}', '!', filtered_output)
 else:
filtered_output = re.sub(r'(\!)\1{2,}', '!!', filtered_output)
 return filtered_output
if real_tweet.startswith(" "):
filtered_output = filtered_output.strip()
text_annotetor = ' '.join(real_tweet.split())
 start = text_annotetor.find(filtered_output)
 end = start + len(filtered_output)
 start -= 0
end += 2
flag = real_tweet.find(" ")
 if flag < start:
filtered_output = real_tweet[start:end]
if " " in real_tweet and not real_tweet.startswith(" "):
filtered_output = filtered_output.strip()
text_annotetor = re.sub(" {2,}", " ", real_tweet)
 start = text_annotetor.find(filtered_output)
 end = start + len(filtered_output)
 start -= 0
end += 2
flag = real_tweet.find(" ")
 if flag < start:
filtered_output = real_tweet[start:end]
return filtered_output<split>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
def eval_jacc(model, dev, decoder, testloader, criterion, dataframe, enable_log=True, is_test=False, every_batch=10):
model.eval()
model.to(dev)
 instance_count = 0
test_loss = 0.0
jacc_score_overall = 0.0
pred_seltext_global = []
batch_textid_global = []
batch_text_global = []
sentiment_global = []
for batchidx, batch in enumerate(testloader):
pred_seltext = []
select_text = []
if batchidx % every_batch == every_batch - 1:
print(f'Dumping eval of {batchidx + 1}-th batch...')
 if not is_test:
testx, testy = batch
testx, testy = testx.to(dev), testy.to(dev)
 else:
testx = batch[0]
testx = testx.to(dev)
 test_offsets = testx[:,:, -2:]
batch_textid_tensor = testx[:, :, 3].to('cpu' ).long()
batch_textid = [decoder(batch_textid_tensor[i,:])for i in range(batch_textid_tensor.shape[0])]
batch_textid_global = batch_textid_global + batch_textid
sentiment_text = [decoder(testx[i, :3, 0].to('cpu' ).long())for i in range(batch_textid_tensor.shape[0])]
sentiment_global = sentiment_global + sentiment_text
batch_text = [str(" " + ' '.join(str(dataframe.loc[dataframe['textID'] == batch_textid[i]]['text'].item() ).split())) for i in range(len(batch_textid)) ]
batch_text_global = batch_text_global + batch_text
if not is_test:
batch_gnd_seltext = [str(dataframe.loc[dataframe['textID'] == batch_textid[i]]['selected_text'].item())for i in range(len(batch_textid)) ]
for idx, item in enumerate(batch_text):
sel_idstart = test_offsets[idx, testy[idx, 0].item() , 0].item()
sel_idend = test_offsets[idx, testy[idx, 1].item() , 1].item()
select_text = select_text + [batch_gnd_seltext[idx]]
with torch.no_grad() :
testoutput = model(testx[:, :, 0], testx[:, :, 1], testx[:, :, 2])
 _, pred_starts = testoutput[:, :, 0].squeeze(dim=-1 ).topk(1, dim=-1)
 _, pred_ends = testoutput[:, :, 1].squeeze(dim=-1 ).topk(1, dim=-1)
 for idx, item in enumerate(batch_text):
instance_count += 1
idstart = test_offsets[idx, pred_starts[idx].item() , 0].item()
idend = test_offsets[idx, pred_ends[idx].item() , 1].item()
if len(item[idstart:idend])== 0:
pred_seltext.append(pp(item,item))
else:
pred_seltext.append(pp(item[idstart:idend],item))
if enable_log:
print('--------------------------------------------------------------------------------')
 print('startidx\t\t\t: {0}'.format(pred_starts[idx].item()))
print('Sentiment\t\t\t: {0}'.format(sentiment_text[idx]))
print('Text\t\t\t\t: {0}'.format(item))
print('Predicted sentiment text\t: {0}'.format(pred_seltext[-1]))
print('Predicted offset tensor\t\t: [{0}, {1}]'.format(pred_starts[idx].item() ,pred_ends[idx].item()))
print('Offset tensors\t\t\t: {0}'.format(test_offsets[idx, :50, :].T))
if not is_test and enable_log:
print('Actual Offset tensors\t\t: {0}'.format(testy[idx, :]))
print('Actual sentiment text\t\t: {0}'.format(batch_gnd_seltext[idx]))
print('Jaccard score\t\t\t: {0}'.format(jaccard(pred_seltext[-1], batch_gnd_seltext[idx])))
 print('--------------------------------------------------------------------------------')
 pred_seltext_global = pred_seltext_global + pred_seltext
if not is_test:
jacc_score = 0.0
for idx, seltext in enumerate(pred_seltext):
jacc_score += jaccard(seltext,select_text[idx])
 jacc_score_overall += jacc_score
if batchidx % every_batch == every_batch - 1:
print(f'Jaccard avg score of {batchidx + 1}-th batch: {jacc_score/len(pred_seltext)}')
 if not is_test:
return {'pred': pred_seltext_global,
'textID': batch_textid_global,
'text': batch_text_global,
'predstart': pred_starts,
'predend': pred_ends,
'jaccard': jacc_score_overall/instance_count,
'sentiment' : sentiment_global}
else:
return {'pred': pred_seltext_global,
'textID': batch_textid_global,
'sentiment' : sentiment_global}<define_variables>
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
small_val_loader =()
for i, val in enumerate(val_loader):
if i < 1:
small_val_loader = small_val_loader +(val,)
 print(str(len(small_val_loader)) + " batch(es)selected!" )<compute_train_metric>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
eval_result = eval_jacc(testmodel, dev, robdist_detokenize, small_val_loader, criterion_mod, train_dataframe, enable_log=True, is_test=False, every_batch=50 )<create_dataframe>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
print("-"* 20 + "Fold :"+str(fold_)+ "-"* 20)
 trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=1000)
 reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 print("-"* 20 + "Xgboost Training" + "-"* 20)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds1[val_] = xg.predict(val_x)
 oof_preds[oof_preds < 0] = 0
oof_preds1[oof_preds1 < 0] = 0
both_oof[val_] = oof_preds[val_] * 0.6 + oof_preds1[val_] * 0.4
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_data.columns])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
print("LGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5)
 print("XGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1)**.5)
 print("Combine ", mean_squared_error(np.log1p(trn_user_target['target']), both_oof)**.5 )
test_x_ids = stackup_ids(test_preprocess['BPEencoded'],
test_preprocess['att_mask'],
test_preprocess['tok_typeids'],
test_preprocess['textID'],
test_preprocess['offsets'])
 test_x_ids_dataset = TensorDataset(test_x_ids)
 test_loader = DataLoader(test_x_ids_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=1)
<save_to_csv>
tips = 2
testeval_result = eval_jacc(testmodel, dev, robdist_detokenize, test_loader, criterion_mod, test_dataframe, enable_log=False, is_test=True, every_batch=10)
 testoutdataframe = pd.DataFrame(data={'textID': testeval_result['textID'],
'selected_text': testeval_result['pred'],
'sentiment': testeval_result['sentiment']})
 testoutdataframe.to_csv(path_or_buf=os.path.join('/kaggle/working', 'testdataset_out.csv'), index=False)
 submission = pd.DataFrame(data={'textID': testeval_result['textID'],
'selected_text': testeval_result['pred']})
 submission.to_csv(path_or_buf=os.path.join('/kaggle/working', 'submission.csv'), index=False )<set_options>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )