Kaggle competition: Google Analytics Customer Revenue Prediction
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 try:
return float(len(c)) /(len(a)+ len(b)- len(c))
except:
return 0<predict_on_test>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str)
 train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False)
 for df in [train, test]:
df["visitId"] = df["visitId"].apply(lambda x: x.split('.', 1)[0] ).astype(str)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId")
 for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 for df in [train_new, test_new]:
df["Sessions"] = df["Sessions"].fillna(0)
 df["Avg.Session Duration"] = df["Avg.Session Duration"].fillna(0)
 df["Bounce Rate"] = df["Bounce Rate"].fillna(0)
 df["Revenue"] = df["Revenue"].fillna(0)
 df["Transactions"] = df["Transactions"].fillna(0)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].fillna(0)
 df['trafficSource.adContent'].fillna('N/A', inplace=True)
 df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)
 df['trafficSource.referralPath'].fillna('N/A', inplace=True)
 df['trafficSource.keyword'].fillna('N/A', inplace=True)
 df['totals.bounces'].fillna(0.0, inplace=True)
 df['totals.newVisits'].fillna(0.0, inplace=True)
 df['totals.pageviews'].fillna(0.0, inplace=True)
 del train
del test
train = train_new
test = test_new
del train_new
del test_new
gc.collect()
def predict(model, dataset, batch_size=32):
model.eval()
start_probas = []
end_probas = []
loader = DataLoader(
 dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS 
)
 with torch.no_grad() :
for data in loader:
start_logits, end_logits = model(
 data["ids"].cuda() ,
data['sentiment_input'].cuda() ,
data['probas_start'].cuda() ,
data['probas_end'].cuda() 
)
 start_probs = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_probs = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for s, e in zip(start_probs, end_probs):
start_probas.append(list(s))
end_probas.append(list(e))
return start_probas, end_probas<choose_model_class>
for df in [train, test]:
df.rename({'fullVisitorId': 'id', 'totals.transactionRevenue': 'target'}, axis = 1, inplace = True)
 df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['weekday'] = df['date'].dt.dayofweek
df['hour'] = df['date'].dt.hour
df['monthday'] = df['date'].dt.day
df.sort_values(['id', 'date'], ascending = True, inplace = True)
 df['next_session'] =(df['date'] - df.groupby('id', sort = False)['date'].shift(1)).astype(np.int64)// 1e9 // 60 // 60
df['prev_session'] =(df['date'] - df.groupby('id', sort = False)['date'].shift(-1)).astype(np.int64)// 1e9 // 60 // 60
df.sort_index(inplace = True)
 train['target'].fillna(0, inplace = True)
 user_labels =(train.groupby('id', sort = False)['target'].max() > 0 ).astype(int)
 user_sums = np.log1p(np.array(train.groupby('id', sort = False)['target'].sum() ).tolist())
 user_ids = train['id'].unique()
session_sums = train['target'].copy()
del train['target']
class SWA(Optimizer):
def __init__(self, optimizer, swa_start=None, swa_freq=None, swa_lr=None):
r
self._auto_mode,(self.swa_start, self.swa_freq)= \
self._check_params(self, swa_start, swa_freq)
 self.swa_lr = swa_lr
if self._auto_mode:
if swa_start < 0:
raise ValueError("Invalid swa_start: {}".format(swa_start))
if swa_freq < 1:
raise ValueError("Invalid swa_freq: {}".format(swa_freq))
else:
if self.swa_lr is not None:
warnings.warn(
 "Some of swa_start, swa_freq is None, ignoring swa_lr")
 self.swa_lr = None
self.swa_start = None
self.swa_freq = None
if self.swa_lr is not None and self.swa_lr < 0:
raise ValueError("Invalid SWA learning rate: {}".format(swa_lr))
self.optimizer = optimizer
self.defaults = self.optimizer.defaults
self.param_groups = self.optimizer.param_groups
self.state = defaultdict(dict)
 self.opt_state = self.optimizer.state
for group in self.param_groups:
group['n_avg'] = 0
group['step_counter'] = 0
@staticmethod
def _check_params(self, swa_start, swa_freq):
params = [swa_start, swa_freq]
params_none = [param is None for param in params]
if not all(params_none)and any(params_none):
warnings.warn(
 "Some of swa_start, swa_freq is None, ignoring other")
 for i, param in enumerate(params):
if param is not None and not isinstance(param, int):
params[i] = int(param)
 warnings.warn("Casting swa_start, swa_freq to int")
 return not any(params_none), params
def _reset_lr_to_swa(self):
if self.swa_lr is None:
return
for param_group in self.param_groups:
if param_group['step_counter'] >= self.swa_start:
param_group['lr'] = self.swa_lr
def update_swa_group(self, group):
r
for p in group['params']:
param_state = self.state[p]
if 'swa_buffer' not in param_state:
param_state['swa_buffer'] = torch.zeros_like(p.data)
 buf = param_state['swa_buffer']
virtual_decay = 1 / float(group["n_avg"] + 1)
 diff =(p.data - buf)* virtual_decay
buf.add_(diff)
 group["n_avg"] += 1
def update_swa(self):
r
for group in self.param_groups:
self.update_swa_group(group)
 def swap_swa_sgd(self):
r
for group in self.param_groups:
for p in group['params']:
param_state = self.state[p]
if 'swa_buffer' not in param_state:
warnings.warn(
 "SWA wasn't applied to param {}; skipping it".format(p))
continue
buf = param_state['swa_buffer']
tmp = torch.empty_like(p.data)
 tmp.copy_(p.data)
 p.data.copy_(buf)
 buf.copy_(tmp)
 def step(self, closure=None):
r
self._reset_lr_to_swa()
loss = self.optimizer.step(closure)
 for group in self.param_groups:
group["step_counter"] += 1
steps = group["step_counter"]
if self._auto_mode:
if steps > self.swa_start and steps % self.swa_freq == 0:
self.update_swa_group(group)
 return loss
def state_dict(self):
r
opt_state_dict = self.optimizer.state_dict()
swa_state = {(id(k)if isinstance(k, torch.Tensor)else k): v
for k, v in self.state.items() }
opt_state = opt_state_dict["state"]
param_groups = opt_state_dict["param_groups"]
return {"opt_state": opt_state, "swa_state": swa_state,
"param_groups": param_groups}
def load_state_dict(self, state_dict):
r
swa_state_dict = {"state": state_dict["swa_state"],
"param_groups": state_dict["param_groups"]}
opt_state_dict = {"state": state_dict["opt_state"],
"param_groups": state_dict["param_groups"]}
super(SWA, self ).load_state_dict(swa_state_dict)
 self.optimizer.load_state_dict(opt_state_dict)
 self.opt_state = self.optimizer.state
def add_param_group(self, param_group):
r
param_group['n_avg'] = 0
param_group['step_counter'] = 0
self.optimizer.add_param_group(param_group)
 @staticmethod
def bn_update(loader, model, device=None):
r
if not _check_bn(model):
return
was_training = model.training
model.train()
momenta = {}
model.apply(_reset_bn)
 model.apply(lambda module: _get_momenta(module, momenta))
n = 0
for input in loader:
if isinstance(input,(list, tuple)) :
input = input[0]
b = input.size(0)
 momentum = b / float(n + b)
 for module in momenta.keys() :
module.momentum = momentum
if device is not None:
input = input.to(device)
 model(input)
 n += b
model.apply(lambda module: _set_momenta(module, momenta))
model.train(was_training)
 def _check_bn_apply(module, flag):
if issubclass(module.__class__, torch.nn.modules.batchnorm._BatchNorm):
flag[0] = True
def _check_bn(model):
flag = [False]
model.apply(lambda module: _check_bn_apply(module, flag))
return flag[0]
def _reset_bn(module):
if issubclass(module.__class__, torch.nn.modules.batchnorm._BatchNorm):
module.running_mean = torch.zeros_like(module.running_mean)
 module.running_var = torch.ones_like(module.running_var)
 def _get_momenta(module, momenta):
if issubclass(module.__class__, torch.nn.modules.batchnorm._BatchNorm):
momenta[module] = module.momentum
def _set_momenta(module, momenta):
if issubclass(module.__class__, torch.nn.modules.batchnorm._BatchNorm):
module.momentum = momenta[module]<train_model>
mobile_words = {'android', 'samsung', 'mini', 'iphone', 'in-app', 'playstation',
'mozilla', 'chrome', 'blackberry', 'nokia', 'browser', 'amazon',
'lunascape', 'netscape', 'konqueror', 'puffin', 'amazon'}
normal_browsers = {'chrome', 'safari', 'firefox', 'internet explorer', 'edge', 'opera',
'coc coc', 'maxthon', 'iron'}
key_sources = {'google', 'youtube', 'yahoo', 'facebook', 'reddit', 'bing', 'outlook', 'linkedin',
'pinterest', 'ask', 'siliconvalley', 'lunametrics', 'amazon', 'mysearch', 'qiita',
'messenger', 'twitter', 't.co', 'vk.com', 'search', 'edu', 'mail', 'ad', 'golang',
'direct', 'dealspotr', 'sashihara', 'phandroid', 'baidu', 'mdn', 'duckduckgo', 'seroundtable',
'metrics', 'sogou', 'businessinsider', 'github', 'gophergala', 'yandex', 'msn', 'dfa',
'feedly', 'arstechnica', 'squishable', 'flipboard', 't-online.de', 'sm.cn', 'wow', 'baidu',
'partners'}
def browser_mapping(x):
if x in normal_browsers:
return x
elif any([word in x for word in mobile_words]):
return 'mobile_browser'
elif '(not set)' in x:
return 'nan'
else:
return 'others'
def adcontents_mapping(x):
if 'google' in x:
return 'google'
elif '(not set)' in x or 'nan' in x:
return 'nan'
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
for word in key_sources:
if word in x:
return word
if '(not set)' in x or 'nan' in x:
return 'nan'
else:
return 'others'
for df in [train, test]:
df['device.browser'] = df['device.browser'].astype(str ).map(lambda x: browser_mapping(x.lower()))
df['trafficSource.adContent'] = df['trafficSource.adContent'].astype(str ).map(lambda x: browser_mapping(x.lower()))
df['trafficSource.source'] = df['trafficSource.source'].astype(str ).map(lambda x: source_mapping(x.lower()))
pairs = [('trafficSource.source', 'geoNetwork.country'),('trafficSource.campaign', 'trafficSource.medium'), 
('device.browser', 'device.deviceCategory'),('device.browser', 'device.operatingSystem'), 
('device.browser', 'channelGrouping'),('device.deviceCategory', 'channelGrouping'), 
('device.operatingSystem', 'channelGrouping'), 
('trafficSource.adContent', 'source_country'),('trafficSource.medium', 'source_country')]
def get_second_part(word):
return re.sub('.*\.', '', word)
 for df in [train, test]:
for first, second in pairs:
df[get_second_part(first)+ '_' + get_second_part(second)] = df[first] + '_' + df[second]
for first in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro',
'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for second in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[get_second_part(first)+ "_" + get_second_part(second)] = df[first] + "_" + df[second]
def fit(
 model,
train_dataset,
val_dataset,
loss_config,
epochs=5,
swa_first_epoch=5,
batch_size=8,
acc_steps=1,
weight_decay=0,
warmup_prop=0.0,
lr=5e-4,
cp=False,
use_len_sampler=True, 
):
best_jac = 0
train_loader = DataLoader(
 train_dataset, batch_size=batch_size, num_workers=NUM_WORKERS 
)
 val_loader = DataLoader(
 val_dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS 
)
 optimizer = Adam(model.parameters() , lr=lr)
 optimizer = SWA(optimizer)
 n_steps = float(epochs * len(train_loader)) / float(acc_steps)
 num_warmup_steps = int(warmup_prop * n_steps)
 scheduler = get_linear_schedule_with_warmup(
 optimizer, num_warmup_steps, n_steps 
)
 total_steps = 0
for epoch in range(epochs):
model.train()
start_time = time.time()
optimizer.zero_grad()
avg_loss = 0
for step, data in enumerate(train_loader):
total_steps += 1
start_logits, end_logits = model(
 data["ids"].cuda() ,
data['sentiment_input'].cuda() ,
data['probas_start'].cuda() ,
data['probas_end'].cuda() 
)
 loss = loss_fn(
 start_logits,
end_logits,
data["target_start"].cuda() ,
data["target_end"].cuda() ,
config=loss_config, 
)
 avg_loss += loss.item() / len(train_loader)
 loss.backward()
if(step + 1)% acc_steps == 0:
optimizer.step()
scheduler.step()
model.zero_grad()
model.eval()
avg_val_loss = 0.0
val_jac = 0.0
if epoch + 1 >= swa_first_epoch:
optimizer.update_swa()
optimizer.swap_swa_sgd()
with torch.no_grad() :
for data in val_loader:
start_logits, end_logits = model(
 data["ids"].cuda() ,
data['sentiment_input'].cuda() ,
data['probas_start'].cuda() ,
data['probas_end'].cuda() 
)
 loss = loss_fn(
 start_logits.detach() ,
end_logits.detach() ,
data["target_start"].cuda().detach() ,
data["target_end"].cuda().detach() ,
config=loss_config, 
)
 avg_val_loss += loss.item() / len(val_loader)
 val_jac += jaccard_from_logits_string(data, start_logits, end_logits)/ len(
 val_dataset 
)
 if epoch + 1 >= swa_first_epoch:
optimizer.swap_swa_sgd()
if val_jac >= best_jac and cp:
save_model_weights(model, "checkpoint.pt", verbose=0)
 best_jac = val_jac
dt = time.time() - start_time
lr = scheduler.get_lr() [0]
print(f"Epoch {epoch + 1}/{epochs} \t lr={lr:.1e} \t t={dt:.0f}s \t", end="")
 print(
 f"loss={avg_loss:.3f} \t val_loss={avg_val_loss:.3f} \t val_jaccard={val_jac:.4f}" 
)
 del loss, data, avg_val_loss, avg_loss, train_loader, val_loader
torch.cuda.empty_cache()
gc.collect()
if epoch >= swa_first_epoch:
optimizer.swap_swa_sgd()
return best_jac if cp else val_jac<choose_model_class>
excluded_cols = {'date', 'id', 'visitId', 'visitStartTime', 'sessionId'}
cat_cols = [col for col in train.columns if col not in excluded_cols and train[col].dtype == 'object']
for col in cat_cols:
train[col], indexer = pd.factorize(train[col])
 test[col] = indexer.get_indexer(test[col] )
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, dilation=1, padding="same", use_bn=True):
super().__init__()
if padding == "same":
padding = kernel_size // 2 * dilation
if use_bn:
self.conv = nn.Sequential(
 nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, stride=stride, dilation=dilation),
nn.BatchNorm1d(out_channels),
nn.ReLU() , 
)
 else:
self.conv = nn.Sequential(
 nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding, stride=stride, dilation=dilation),
nn.ReLU() , 
)
 def forward(self, x):
return self.conv(x)
 class Waveblock(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=3, dilations=[1], padding="same"):
super().__init__()
self.n = len(dilations)
 if padding == "same":
padding = kernel_size // 2
self.init_conv = nn.Conv1d(in_channels, out_channels, 1)
 self.convs_tanh = nn.ModuleList([])
 self.convs_sigm = nn.ModuleList([])
 self.convs = nn.ModuleList([])
 for dilation in dilations:
self.convs_tanh.append(
 nn.Sequential(
 nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding*dilation, dilation=dilation),
nn.Tanh() , 
)
)
 self.convs_sigm.append(
 nn.Sequential(
 nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding*dilation, dilation=dilation),
nn.Sigmoid() , 
)
)
 self.convs.append(nn.Conv1d(out_channels, out_channels, 1))
def forward(self, x):
x = self.init_conv(x)
 res_x = x
for i in range(self.n):
x_tanh = self.convs_tanh[i](x)
 x_sigm = self.convs_sigm[i](x)
 x = x_tanh * x_sigm
x = self.convs[i](x)
 res_x = res_x + x
return res_x<choose_model_class>
n_splits = 5
splits = get_folds(df = train, n_splits = n_splits)
 train_cols = [col for col in train.columns if col not in excluded_cols]
oof_preds = np.zeros(train.shape[0])
 test_preds = np.zeros(test.shape[0])
 val_scores = []
for i in range(n_splits):
tr_idx, val_idx = splits[i]
print("Fold:", i + 1, end = '.')
 train_X, train_y = train[train_cols].iloc[tr_idx].values, np.log1p(session_sums[tr_idx])
 val_X, val_y = train[train_cols].iloc[val_idx].values, np.log1p(session_sums[val_idx])
 gbm = LGBMRegressor(num_leaves = 31, learning_rate = 0.03, n_estimators = 1000, subsample=.9,
colsample_bytree=.9 , random_state = 1)
 gbm.fit(train_X, train_y, eval_set =(val_X, val_y),early_stopping_rounds = 150, verbose = False,
eval_metric = 'rmse')
 val_pred = gbm.predict(val_X)
 oof_preds[val_idx] = val_pred
val_scores.append(np.sqrt(mean_squared_error(val_pred, val_y)))
 print('Score:', val_scores[-1])
 test_pred = gbm.predict(test[train_cols])
 test_pred[test_pred < 0] = 0
test_preds += np.expm1(test_pred)/ n_splits
oof_preds[oof_preds < 0] = 0
print(np.sqrt(mean_squared_error(np.log1p(session_sums), oof_preds)))
 train['preds'] = np.expm1(oof_preds)
 train['log_preds'] = oof_preds
test['preds'] = test_preds
test['log_preds'] = np.log1p(test_preds )
class TweetCharModel(nn.Module):
def __init__(self, len_voc, use_msd=True,
embed_dim=64, lstm_dim=64, char_embed_dim=32, sent_embed_dim=32, ft_lstm_dim=32, n_models=1):
super().__init__()
self.use_msd = use_msd
self.char_embeddings = nn.Embedding(len_voc, char_embed_dim)
 self.sentiment_embeddings = nn.Embedding(3, sent_embed_dim)
 self.proba_lstm = nn.LSTM(n_models * 2, ft_lstm_dim, batch_first=True, bidirectional=True)
 self.lstm = nn.LSTM(char_embed_dim + ft_lstm_dim * 2 + sent_embed_dim, lstm_dim, batch_first=True, bidirectional=True)
 self.lstm2 = nn.LSTM(lstm_dim * 2, lstm_dim, batch_first=True, bidirectional=True)
 self.logits = nn.Sequential(
 nn.Linear(lstm_dim * 4, lstm_dim),
nn.ReLU() ,
nn.Linear(lstm_dim, 2), 
)
 self.high_dropout = nn.Dropout(p=0.5)
 def forward(self, tokens, sentiment, start_probas, end_probas):
bs, T = tokens.size()
probas = torch.cat([start_probas, end_probas], -1)
 probas_fts, _ = self.proba_lstm(probas)
 char_fts = self.char_embeddings(tokens)
 sentiment_fts = self.sentiment_embeddings(sentiment ).view(bs, 1, -1)
 sentiment_fts = sentiment_fts.repeat(( 1, T, 1))
features = torch.cat([char_fts, sentiment_fts, probas_fts], -1)
 features, _ = self.lstm(features)
 features2, _ = self.lstm2(features)
 features = torch.cat([features, features2], -1)
 if self.use_msd and self.training:
logits = torch.mean(
 torch.stack(
 [self.logits(self.high_dropout(features)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.logits(features)
 start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]
return start_logits, end_logits<choose_model_class>
stats = ['max', 'mean', 'median', 'std', 'size', 'sum']
user_train = train[train_cols + ['id']].groupby('id', sort = False ).mean()
user_test = test[train_cols + ['id']].groupby('id', sort = False ).mean()
train_preds = train.groupby('id', sort = False ).agg({'preds': stats, 'log_preds': ['sum']} ).fillna(0)
 train_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']
for col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:
train_preds[col] = np.log1p(train_preds[col])
 user_train = user_train.merge(train_preds, left_index = True, right_index = True)
 test_preds = test.groupby('id', sort = False ).agg({'preds': stats, 'log_preds': ['sum']} ).fillna(0)
 test_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']
for col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:
test_preds[col] = np.log1p(test_preds[col])
 user_test = user_test.merge(test_preds, left_index = True, right_index = True )
class WaveNet(nn.Module):
def __init__(self, len_voc, use_msd=True, dilations=[1],
cnn_dim=64, char_embed_dim=32, sent_embed_dim=32, proba_cnn_dim=32, n_models=1, kernel_size=3, use_bn=True):
super().__init__()
self.use_msd = use_msd
self.char_embeddings = nn.Embedding(len_voc, char_embed_dim)
 self.sentiment_embeddings = nn.Embedding(3, sent_embed_dim)
 self.probas_cnn = ConvBlock(n_models * 2, proba_cnn_dim, kernel_size=kernel_size, use_bn=use_bn)
 self.cnn = nn.Sequential(
 Waveblock(char_embed_dim + sent_embed_dim + proba_cnn_dim, cnn_dim, kernel_size=kernel_size, dilations=dilations),
nn.BatchNorm1d(cnn_dim),
Waveblock(cnn_dim, cnn_dim * 2, kernel_size=kernel_size, dilations=dilations),
nn.BatchNorm1d(cnn_dim * 2),
Waveblock(cnn_dim * 2 , cnn_dim * 4, kernel_size=kernel_size, dilations=dilations),
nn.BatchNorm1d(cnn_dim * 4), 
)
 self.logits = nn.Sequential(
 nn.Linear(cnn_dim * 4, cnn_dim),
nn.ReLU() ,
nn.Linear(cnn_dim, 2), 
)
 self.high_dropout = nn.Dropout(p=0.5)
 def forward(self, tokens, sentiment, start_probas, end_probas):
bs, T = tokens.size()
probas = torch.cat([start_probas, end_probas], -1 ).permute(0, 2, 1)
 probas_fts = self.probas_cnn(probas ).permute(0, 2, 1)
 char_fts = self.char_embeddings(tokens)
 sentiment_fts = self.sentiment_embeddings(sentiment ).view(bs, 1, -1)
 sentiment_fts = sentiment_fts.repeat(( 1, T, 1))
x = torch.cat([char_fts, sentiment_fts, probas_fts], -1 ).permute(0, 2, 1)
 features = self.cnn(x ).permute(0, 2, 1)
 if self.use_msd and self.training:
logits = torch.mean(
 torch.stack(
 [self.logits(self.high_dropout(features)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.logits(features)
 start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]
return start_logits, end_logits<choose_model_class>
time_min = train['visitStartTime'].min()
time_max = train['visitStartTime'].max()
for df in [train, test]:
df['visitStartTime'] -= time_min
df['visitStartTime'] /=(time_max - time_min)
 aggregations = ['min', 'max', 'std']
times_train = train.groupby('id', sort = False)['visitStartTime'].agg(aggregations ).fillna(0)
 times_train.columns = ['times_' + word for word in aggregations]
times_train['times_diff'] = times_train['times_max'] - times_train['times_min']
times_train['times_diff_n'] = times_train['times_diff'] / user_train['pred_size']
times_train.drop(['times_min', 'times_max'], axis = 1, inplace = True)
 times_test = test.groupby('id', sort = False)['visitStartTime'].agg(aggregations ).fillna(0)
 times_test.columns = ['times_' + word for word in aggregations]
times_test['times_diff'] = times_test['times_max'] - times_test['times_min']
times_test['times_diff_n'] = times_test['times_diff'] / user_test['pred_size']
times_test.drop(['times_min', 'times_max'], axis = 1, inplace = True)
 train['date'] = train['date'].astype(str ).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))
 test['date'] = test['date'].astype(str ).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))
 def analyze_dates(user):
features = []
dates = sorted(user['date'].values)
 n = user.shape[0]
diff =(dates[-1] - dates[0] ).days/360
features += [diff, diff/n]
features += [Counter(dates ).most_common() [0][1]]
return features
dates_train = np.array(train.groupby('id', sort = False ).apply(lambda x: analyze_dates(x)).tolist())
 dates_test = np.array(test.groupby('id', sort = False ).apply(lambda x: analyze_dates(x)).tolist() )
class ConvNet(nn.Module):
def __init__(self, len_voc, use_msd=True,
cnn_dim=64, char_embed_dim=32, sent_embed_dim=32, proba_cnn_dim=32, n_models=1, kernel_size=3, use_bn=False):
super().__init__()
self.use_msd = use_msd
self.char_embeddings = nn.Embedding(len_voc, char_embed_dim)
 self.sentiment_embeddings = nn.Embedding(3, sent_embed_dim)
 self.probas_cnn = ConvBlock(n_models * 2, proba_cnn_dim, kernel_size=kernel_size, use_bn=use_bn)
 self.cnn = nn.Sequential(
 ConvBlock(char_embed_dim + sent_embed_dim + proba_cnn_dim, cnn_dim, kernel_size=kernel_size, use_bn=use_bn),
ConvBlock(cnn_dim, cnn_dim * 2, kernel_size=kernel_size, use_bn=use_bn),
ConvBlock(cnn_dim * 2 , cnn_dim * 4, kernel_size=kernel_size, use_bn=use_bn),
ConvBlock(cnn_dim * 4, cnn_dim * 8, kernel_size=kernel_size, use_bn=use_bn), 
)
 self.logits = nn.Sequential(
 nn.Linear(cnn_dim * 8, cnn_dim),
nn.ReLU() ,
nn.Linear(cnn_dim, 2), 
)
 self.high_dropout = nn.Dropout(p=0.5)
 def forward(self, tokens, sentiment, start_probas, end_probas):
bs, T = tokens.size()
probas = torch.cat([start_probas, end_probas], -1 ).permute(0, 2, 1)
 probas_fts = self.probas_cnn(probas ).permute(0, 2, 1)
 char_fts = self.char_embeddings(tokens)
 sentiment_fts = self.sentiment_embeddings(sentiment ).view(bs, 1, -1)
 sentiment_fts = sentiment_fts.repeat(( 1, T, 1))
x = torch.cat([char_fts, sentiment_fts, probas_fts], -1 ).permute(0, 2, 1)
 features = self.cnn(x ).permute(0, 2, 1)
 if self.use_msd and self.training:
logits = torch.mean(
 torch.stack(
 [self.logits(self.high_dropout(features)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.logits(features)
 start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]
return start_logits, end_logits<split>
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.03,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
lgb_params = {
'learning_rate': 0.03,
'metric': 'rmse',
'subsample': 0.9,
'colsample_bytree': 0.9,
'random_state': 1,
'num_leaves': 31
}
lgb_params_2 = {
'learning_rate': 0.03,
'metric': 'rmse',
'subsample': 0.9,
'colsample_bytree': 0.9,
'random_state': 1,
'num_leaves': 10
}
def k_fold(config, df_train, df_test, X_train, X_test, preds, len_voc, k=5, seed=42, save=True, model_name='model'):
time = str(datetime.datetime.now())[:16]
score = 0
splits = list(StratifiedKFold(n_splits=k, random_state=seed ).split(X=df_train, y=df_train['sentiment']))
pred_oof = [[[], []] for i in range(len(df_train)) ]
pred_tests = []
test_dataset = TweetCharDataset(
 df_test, X_test, preds['test_start'], preds['test_end'],
max_len=config.max_len_val, train=False, n_models=config.n_models 
)
 for i,(train_idx, val_idx)in enumerate(splits):
print(f"
------------- Fold {i + 1} -------------")
 seed_everything(seed + i)
 if config.model == 'rnn':
model = TweetCharModel(
 len_voc,
use_msd=config.use_msd,
n_models=config.n_models,
lstm_dim=config.lstm_dim,
ft_lstm_dim=config.ft_lstm_dim,
char_embed_dim=config.char_embed_dim,
sent_embed_dim=config.sent_embed_dim,
).cuda()
elif config.model == 'cnn':
model = ConvNet(
 len_voc,
use_msd=config.use_msd,
n_models=config.n_models,
use_bn=config.use_bn,
cnn_dim=config.cnn_dim,
proba_cnn_dim=config.proba_cnn_dim,
char_embed_dim=config.char_embed_dim,
sent_embed_dim=config.sent_embed_dim,
kernel_size=config.kernel_size,
).cuda()
else:
model = WaveNet(
 len_voc,
use_msd=config.use_msd,
n_models=config.n_models,
use_bn=config.use_bn,
cnn_dim=config.cnn_dim,
proba_cnn_dim=config.proba_cnn_dim,
char_embed_dim=config.char_embed_dim,
sent_embed_dim=config.sent_embed_dim,
kernel_size=config.kernel_size,
dilations=config.dilations,
).cuda()
model.zero_grad()
train_dataset = TweetCharDataset(
 df_train.iloc[train_idx],
X_train[train_idx],
preds['oof_start'][train_idx],
preds['oof_end'][train_idx],
max_len=config.max_len,
n_models=config.n_models, 
)
 val_dataset = TweetCharDataset(
 df_train.iloc[val_idx],
X_train[val_idx],
preds['oof_start'][val_idx],
preds['oof_end'][val_idx],
max_len=config.max_len_val,
n_models=config.n_models, 
)
 print('
- Training
')
 fold_score = fit(
 model,
train_dataset,
val_dataset,
config.loss_config,
epochs=config.epochs,
batch_size=config.batch_size,
lr=config.lr,
warmup_prop=config.warmup_prop,
swa_first_epoch=config.swa_first_epoch,
use_len_sampler=config.use_len_sampler,
cp=False 
)
 score += fold_score / k
print('
- Predicting ')
 pred_val_start, pred_val_end = predict(model, val_dataset, batch_size=config.batch_size_val)
 for j, idx in enumerate(val_idx):
pred_oof[idx] = [pred_val_start[j], pred_val_end[j]]
pred_test = predict(model, test_dataset, batch_size=config.batch_size_val)
 pred_tests.append(pred_test)
 del model, train_dataset, val_dataset
torch.cuda.empty_cache()
gc.collect()
print(f'
Local CV jaccard is {score:.4f}')
 return pred_oof, pred_tests<init_hyperparams>
splits = get_folds(df = user_train.reset_index() , n_splits = n_splits)
 oof_preds = {'lgb': np.zeros(user_train.shape[0]),
'xgb': np.zeros(user_train.shape[0]),
'weighted': np.zeros(user_train.shape[0])}
sub_preds = {'lgb': np.zeros(user_test.shape[0]),
'xgb': np.zeros(user_test.shape[0])}
val_scores = {'lgb': [], 'xgb': [], 'weighted': []}
print(' fold | lgb | xgb | weighted ')
 print('---------------------------------------')
 for i in range(n_splits):
tr, val = splits[i]
train_X, train_y = np.hstack([user_train.iloc[tr], dates_train[tr], times_train.iloc[tr]]), user_sums[tr]
val_X, val_y = np.hstack([user_train.iloc[val], dates_train[val], times_train.iloc[val]]), user_sums[val]
models = {'lgb': LGBMRegressor(**lgb_params, n_estimators = 1500),
'xgb': XGBRegressor(**xgb_params, n_estimators = 1000)}
for name in ['xgb', 'lgb']:
models[name].fit(train_X, train_y, eval_set = [(val_X, val_y)],
early_stopping_rounds = 100, eval_metric = 'rmse', verbose = False)
 val_pred = models[name].predict(val_X)
 oof_preds[name][val] = val_pred
val_scores[name].append(np.sqrt(mean_squared_error(val_pred, val_y)))
 test_pred = models[name].predict(np.hstack([user_test, dates_test, times_test]))
test_pred[test_pred < 0] = 0
sub_preds[name] += test_pred / n_splits
val_pred = 0.7 * oof_preds['lgb'][val] + 0.3 * oof_preds['xgb'][val]
oof_preds['weighted'][val] = val_pred
val_scores['weighted'].append(np.sqrt(mean_squared_error(val_pred, val_y)))
 print(' {fold: 3d} | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\ 
.format(fold = i + 1, lgb = val_scores['lgb'][-1], xgb = val_scores['xgb'][-1], w = val_scores['weighted'][-1]))
print('---------------------------------------')
 cv_scores = {}
for name in ['lgb', 'xgb']:
oof_preds[name][oof_preds[name] < 0] = 0
cv_scores[name] = mean_squared_error(user_sums, oof_preds[name])**.5
cv_scores['weighted'] = np.sqrt(mean_squared_error(user_sums, 0.6 * oof_preds['lgb'] + 0.4 * oof_preds['xgb']))
print(' CV | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\ 
.format(lgb = cv_scores['lgb'], xgb = cv_scores['xgb'], w = cv_scores['weighted']))
<init_hyperparams><EOS> 
sub = pd.DataFrame()
sub['fullVisitorId'] = user_test.index
sub['PredictedLogRevenue'] = sub_preds['lgb'] * 0.6 + sub_preds['xgb'] * 0.4
sub.loc[(test.groupby('id', sort = False)['totals.bounces'].min() == 1 ).values, 'PredictedLogRevenue'] = 0.
 sub.to_csv("sub.csv", index = False )