Kaggle competition: Google Analytics Customer Revenue Prediction
util.set_seed(SEED )<load_from_csv>
gc.enable()
features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\
'visitNumber', 'visitStartTime', 'device.browser',\
'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\
'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\
'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\
'geoNetwork.subContinent', 'totals.bounces', 'totals.hits',\
'totals.newVisits', 'totals.pageviews', 'totals.transactionRevenue',\
'trafficSource.adContent', 'trafficSource.campaign',\
'trafficSource.isTrueDirect', 'trafficSource.keyword',\
'trafficSource.medium', 'trafficSource.referralPath',\
'trafficSource.source', 'customDimensions']
def load_df(csv_path):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
ans = pd.DataFrame()
dfs = pd.read_csv(csv_path, sep=',',
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize=100000)
 for df in dfs:
df.reset_index(drop=True, inplace=True)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 use_df = df[features]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis=0 ).reset_index(drop=True)
 return ans
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv' )<import_modules>
%%time
train = load_df('.. /input/train_v2.csv')
 test = load_df('.. /input/test_v2.csv')
 print('train date:', min(train['date']), 'to', max(train['date']))
print('test date:', min(test['date']), 'to', max(test['date']))
import preprocess
import re
import string<categorify>
test['totals.transactionRevenue'] = np.nan
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_df = preprocess.fix_erraneous(train_df, ids_with_target_error, 0 )<drop_column>
all_data = train.append(test, sort=False ).reset_index(drop=True )
def remove_tabs_newlines(text):
text = re.sub(r'\t', ' ', text)
 text = re.sub(r'
', ' ', text)
 return text<feature_engineering>
null_cnt = train.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )
train_df['text'] = train_df['text'].apply(lambda k : remove_tabs_newlines(k))
test_df['text'] = test_df['text'].apply(lambda k : remove_tabs_newlines(k))<drop_column>
for col in ['trafficSource.keyword',
'trafficSource.referralPath',
'trafficSource.adContent']:
all_data[col].fillna('unknown', inplace=True)
 all_data['totals.pageviews'].fillna(1, inplace=True)
 all_data['totals.newVisits'].fillna(0, inplace=True)
 all_data['totals.bounces'].fillna(0, inplace=True)
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int)
 all_data['totals.newVisits'] = all_data['totals.newVisits'].astype(int)
 all_data['totals.bounces'] = all_data['totals.bounces'].astype(int)
 all_data['trafficSource.isTrueDirect'].fillna(False, inplace=True )
def remove_url(text):
return re.sub(r"http\S+", "URL", text )<string_transform>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
def remove_non_ascii(text):
return ''.join([x for x in text if x in string.printable] )<drop_column>
all_data[all_data['fullVisitorId'] == '7813149961404844386'].sort_values(by='visitNumber')[
['date','visitId','visitNumber','totals.hits','totals.pageviews']].head(20 )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )<categorify>
print('unique customDimensions count:', train['customDimensions'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'customDimensions' )
def remove_usrname(text):
text = re.sub(r'@\S{0,}', ' USER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(USER )(\1\b)+', r'\1', text)
 return text<define_variables>
format_str = '%Y%m%d'
all_data['formated_date'] = all_data['date'].apply(lambda x: datetime.strptime(str(x), format_str))
all_data['_year'] = all_data['formated_date'].apply(lambda x:x.year)
 all_data['_month'] = all_data['formated_date'].apply(lambda x:x.month)
 all_data['_quarterMonth'] = all_data['formated_date'].apply(lambda x:x.day//8)
 all_data['_day'] = all_data['formated_date'].apply(lambda x:x.day)
 all_data['_weekday'] = all_data['formated_date'].apply(lambda x:x.weekday())
 all_data.drop(['date','formated_date'], axis=1, inplace=True )
contractions = {
"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'll": "how will",
"how's": "how is",
"i'd": "i would",
"i'll": "i will",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'll": "it will",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"must've": "must have",
"mustn't": "must not",
"needn't": "need not",
"oughtn't": "ought not",
"shan't": "shall not",
"sha'n't": "shall not",
"she'd": "she would",
"she'll": "she will",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"that'd": "that would",
"that's": "that is",
"there'd": "there had",
"there's": "there is",
"they'd": "they would",
"they'll": "they will",
"they're": "they are",
"they've": "they have",
"wasn't": "was not",
"we'd": "we would",
"we'll": "we will",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"where'd": "where did",
"where's": "where is",
"who'll": "who will",
"who's": "who is",
"won't": "will not",
"wouldn't": "would not",
"you'd": "you would",
"you'll": "you will",
"you're": "you are",
"thx" : "thanks"
}
def remove_contractions(text):
return contractions[text.lower() ] if text.lower() in contractions.keys() else text<normalization>
print('train all:', len(train))
print('train unique fullVisitorId:', train['fullVisitorId'].nunique())
 print('train unique visitId:', train['visitId'].nunique())
 print('-' * 30)
 print('test all:', len(test))
print('test unique fullVisitorId:', test['fullVisitorId'].nunique())
 print('test unique visitId:', test['visitId'].nunique())
def remove_punctuations(text):
return text.translate(str.maketrans('', '', string.punctuation))<feature_engineering>
print(all_data['visitNumber'].value_counts() [:5])
 print('-' * 30)
 print(all_data['totals.newVisits'].value_counts())
 print('-' * 30)
 print(all_data['totals.bounces'].value_counts() )
train_df['text'] = train_df['text'].apply(lambda k: remove_usrname(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_contractions(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_url(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_emoji(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_non_ascii(k))
train_df['text'] = train_df['text'].apply(lambda k: remove_punctuations(k))<feature_engineering>
test_df['text'] = test_df['text'].apply(lambda k: remove_usrname(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_contractions(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_url(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_emoji(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_non_ascii(k))
test_df['text'] = test_df['text'].apply(lambda k: remove_punctuations(k))<categorify>
all_data['_visitStartHour'] = all_data['visitStartTime'].apply(
 lambda x: str(datetime.fromtimestamp(x ).hour))
def replace_nums(text):
text = re.sub(r'^\d\S{0,}| \d\S{0,}| \d\S{0,}$', ' NUMBER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(NUMBER )(\1\b)+', r'\1', text)
 text = re.sub(r"[0-9]", " ", text)
 return text<feature_engineering>
print('unique browser count:', train['device.browser'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'device.browser', 10 )
train_df['text'] = train_df['text'].apply(lambda k: replace_nums(k))
test_df['text'] = test_df['text'].apply(lambda k: replace_nums(k))<feature_engineering>
pd.crosstab(all_data['device.deviceCategory'], all_data['device.isMobile'], margins=False)
 all_data['isMobile'] = True
all_data.loc[all_data['device.deviceCategory'] == 'desktop', 'isMobile'] = False
train_df['text'] = train_df['text'].apply(lambda k: cleanup(k))
test_df['text'] = test_df['text'].apply(lambda k: cleanup(k))<prepare_x_and_y>
print('unique operatingSystem count:', train['device.operatingSystem'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'device.operatingSystem', 10 )
train_y = train_df['target']
train_df.drop(['target'], axis=1, inplace=True )<split>
print('unique geoNetwork.city count:', train['geoNetwork.city'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.city', 10 )
X_train, X_test, y_train, y_test \
= train_test_split(train_df['text'], train_y, random_state=SEED, test_size=test_size, stratify=train_y.values )<load_pretrained>
print('unique geoNetwork.region count:', train['geoNetwork.region'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.region', 10 )
tokenizer = transformers.BertTokenizer.from_pretrained(
 model_name,
do_lower_case=True
)<create_dataframe>
print('unique geoNetwork.continent count:', train['geoNetwork.continent'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.continent' )
train_dataset = dataset.BertDataset(
 text=X_train.values,
tokenizer= tokenizer,
max_len= max_len,
target=y_train.values
)<prepare_x_and_y>
print(all_data['totals.hits'].value_counts() [:10])
 all_data['totals.hits'] = all_data['totals.hits'].astype(int )
valid_dataset = dataset.BertDataset(
 text=X_test.values,
tokenizer= tokenizer,
max_len= max_len,
target=y_test.values
)<load_pretrained>
print(all_data['totals.pageviews'].value_counts() [:10])
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int )
train_dl = torch.utils.data.DataLoader(
 train_dataset,
train_bs,
shuffle=True,
num_workers=4
)<load_pretrained>
print('unique trafficSource.adContent count:', train['trafficSource.adContent'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.adContent', 10)
 all_data['_adContentGMC'] =(all_data['trafficSource.adContent'] == 'Google Merchandise Collection' ).astype(np.uint8 )
valid_dl = torch.utils.data.DataLoader(
 valid_dataset,
valid_bs,
shuffle=True,
num_workers=1
)<categorify>
print('unique trafficSource.campaign count:', train['trafficSource.campaign'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.campaign', 10)
 all_data['_withCampaign'] =(all_data['trafficSource.campaign'] != '(not set)' ).astype(np.uint8 )
bert = model.BertHf(model_name, dp=dropout, num_classes=num_classes, linear_in=linear_in)
 bert = bert.to(device )<set_options>
print('unique trafficSource.keyword count:', train['trafficSource.keyword'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.keyword', 10 )
for param in bert.parameters() :
param.requires_grad = True<init_hyperparams>
print('unique trafficSource.medium count:', train['trafficSource.medium'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.medium' )
no_decay = ['bias', 'LayerNorm.weight', 'LayerNorm.bias']
params = list(bert.named_parameters())
 modified_params = [
{'params': [p for n, p in params if not any(nd in n for nd in no_decay)], 'weight_decay': 0.1},
{'params': [p for n, p in params if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]<choose_model_class>
print('unique trafficSource.source count:', train['trafficSource.source'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.source', 10)
 all_data['_sourceGpmall'] =(all_data['trafficSource.source'] == 'mall.googleplex.com' ).astype(np.uint8 )
optim = torch.optim.AdamW(modified_params, lr=start_lr, eps=1e-8 )<define_variables>
_=
all_data['_meanHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('mean')
 all_data['_sumHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('sum')
 for feature in ['totals.hits', 'totals.pageviews']:
info = all_data.groupby('fullVisitorId')[feature].mean()
all_data['_usermean_' + feature] = all_data.fullVisitorId.map(info)
 for feature in ['visitNumber']:
info = all_data.groupby('fullVisitorId')[feature].max()
all_data['_usermax_' + feature] = all_data.fullVisitorId.map(info)
 del info
total_steps = int(len(train_df)* epochs / train_bs)
 warmup_steps = int(len(train_df)* warmup_epochs / train_bs )<normalization>
all_data['_source.country'] = all_data['trafficSource.source'] + '_' + all_data['geoNetwork.country']
all_data['_campaign.medium'] = all_data['trafficSource.campaign'] + '_' + all_data['trafficSource.medium']
all_data['_browser.category'] = all_data['device.browser'] + '_' + all_data['device.deviceCategory']
all_data['_browser.os'] = all_data['device.browser'] + '_' + all_data['device.operatingSystem']
sched = get_linear_schedule_with_warmup(optim, num_warmup_steps=warmup_steps, num_training_steps=total_steps )<choose_model_class>
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )
criterion = torch.nn.CrossEntropyLoss()<compute_train_metric>
all_data.drop(['visitId','visitStartTime'],axis=1,inplace=True)
 for i, t in all_data.loc[:, all_data.columns != 'fullVisitorId'].dtypes.iteritems() :
if t == object:
all_data[i].fillna('unknown', inplace=True)
 all_data[i] = pd.factorize(all_data[i])[0]
train_stat = util.AvgStats()
valid_stat = util.AvgStats()<define_variables>
train = all_data[all_data['totals.transactionRevenue'].notnull() ]
test = all_data[all_data['totals.transactionRevenue'].isnull() ].drop(['totals.transactionRevenue'], axis=1 )
best_acc = 0.0
best_model_file = str(model_name)+ '_best.pth.tar'<train_on_grid>
train_id = train['fullVisitorId']
test_id = test['fullVisitorId']
Y_train_reg = train.pop('totals.transactionRevenue')
 X_train = train.drop(['fullVisitorId'], axis=1)
 X_test = test.drop(['fullVisitorId'], axis=1)
 print(X_train.shape, X_test.shape )
print("
Epoch\tTrain Acc\tTrain Loss\tTrain F1\tValid Acc\tValid Loss\tValid F1")
 for i in range(epochs):
start = time.time()
losses, ops, targs = train.train(train_dl, bert, criterion, optim, sched, device)
 duration = time.time() - start
train_acc = accuracy_score(targs, ops)
 train_f1_score = f1_score(targs, ops)
 train_loss = sum(losses)/len(losses)
 train_prec = precision_score(targs, ops)
 train_rec = recall_score(targs, ops)
 train_roc_auc = roc_auc_score(targs, ops)
 train_stat.append(train_loss, train_acc, train_f1_score, train_prec, train_rec, train_roc_auc, duration)
 start = time.time()
lossesv, opsv, targsv = train.test(valid_dl, bert, criterion, device)
 duration = time.time() - start
valid_acc = accuracy_score(targsv, opsv)
 valid_f1_score = f1_score(targsv, opsv)
 valid_loss = sum(lossesv)/len(lossesv)
 valid_prec = precision_score(targsv, opsv)
 valid_rec = recall_score(targsv, opsv)
 valid_roc_auc = roc_auc_score(targsv, opsv)
 valid_stat.append(valid_loss, valid_acc, valid_f1_score, valid_prec, valid_rec, valid_roc_auc, duration)
 if valid_acc > best_acc:
best_acc = valid_acc
util.save_checkpoint(bert, True, best_model_file)
 tfpr, ttpr, _ = roc_curve(targs, ops)
 train_stat.update_best(ttpr, tfpr, train_acc, i)
 vfpr, vtpr, _ = roc_curve(targsv, opsv)
 valid_stat.update_best(vtpr, vfpr, best_acc, i)
 print("
{}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}\t{:06.8f}".format(i+1, train_acc*100, train_loss,
train_f1_score, valid_acc*100,
valid_loss, valid_f1_score))<compute_test_metric>
del all_data, train, test, train_rev
gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )
print("Summary of best run::")
 print("Best Accuracy: {}".format(valid_stat.best_acc))
print("Roc Auc score: {}".format(valid_stat.roc_aucs[valid_stat.best_epoch]))
print("Loss: {}".format(valid_stat.losses[valid_stat.best_epoch]))
print("Area Under Curve: {}".format(auc(valid_stat.fprs, valid_stat.tprs)) )<create_dataframe>
from sklearn.model_selection import StratifiedKFold, GroupKFold
from sklearn.metrics import roc_auc_score
import lightgbm as lgb
test_dataset = dataset.BertDataset(
 text=test_df.text.values,
tokenizer= tokenizer,
max_len= max_len
)<load_pretrained>
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.6,
'feature_fraction': 0.6
}
folds = GroupKFold(n_splits=5)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train_reg, groups=train_id)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train_reg.iloc[trn_]
val_x, val_y = X_train.iloc[val_], Y_train_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=3000)
 reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=50, verbose=500)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 sub_preds += reg.predict(X_test, num_iteration=reg.best_iteration_)/ folds.n_splits
pred = sub_preds
<load_pretrained><EOS> 
submission = pd.DataFrame({'fullVisitorId':test_id, 'PredictedLogRevenue':pred})
 submission["PredictedLogRevenue"] = np.expm1(submission["PredictedLogRevenue"])
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].apply(lambda x : 0.0 if x < 0 else x)
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].fillna(0.0)
 submission_sum = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId' ).sum().reset_index()
submission_sum["PredictedLogRevenue"] = np.log1p(submission_sum["PredictedLogRevenue"])
 submission_sum.to_csv("submission.csv", index=False)
 submission_sum.head(20 )