Kaggle competition: Google Analytics Customer Revenue Prediction
spell = SpellChecker(distance=1)
 def correctSpelling(oldtext_w, oldtext_w_n = None):
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = spell.correction(current_word)
 new_word = new_word.split()
newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n<train_model>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline
test_string = ["it","is","not","an","englidh","wotd"]
print("Before correctSpelling")
 print(test_string)
 print("After correctSpelling")
 print(correctSpelling(test_string)[0] )<feature_engineering>
gc.enable()
features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\
'visitNumber', 'visitStartTime', 'device.browser',\
'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\
'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\
'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\
'geoNetwork.subContinent', 'totals.bounces', 'totals.hits',\
'totals.newVisits', 'totals.pageviews', 'totals.transactionRevenue',\
'trafficSource.adContent', 'trafficSource.campaign',\
'trafficSource.isTrueDirect', 'trafficSource.keyword',\
'trafficSource.medium', 'trafficSource.referralPath',\
'trafficSource.source', 'customDimensions']
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: correctSpelling(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: correctSpelling(df.selected_w), axis=1))
print("===== E> Fixed 1 char typos <3 =====" )<feature_engineering>
def load_df(csv_path):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
ans = pd.DataFrame()
dfs = pd.read_csv(csv_path, sep=',',
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize=100000)
 for df in dfs:
df.reset_index(drop=True, inplace=True)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 use_df = df[features]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis=0 ).reset_index(drop=True)
 return ans
def addSpace(oldtext_w):
newtext_w = copy.deepcopy(oldtext_w)
 for word_ind ,current_word in enumerate(oldtext_w):
newtext_w[word_ind] = " "+current_word
return newtext_w<feature_engineering>
%%time
train = load_df('/kaggle/input/ga-customer-revenue-prediction/train_v2.csv')
 test = load_df('/kaggle/input/ga-customer-revenue-prediction/test_v2.csv')
 print('train date:', min(train['date']), 'to', max(train['date']))
print('test date:', min(test['date']), 'to', max(test['date']))
dfTrain['text_w'] = dfTrain.apply(lambda df: addSpace(df.text_w), axis=1)
 dfTrain['selected_w'] = dfTrain.apply(lambda df: addSpace(df.selected_w), axis=1)
 print("===== E> Added spaces before every word <3 =====" )<feature_engineering>
def missing_values(data):
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() / data.isnull().count() * 100 ).sort_values(ascending = False)
 df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 print("Total columns at least one Values: ")
 print(df[~(df['Total'] == 0)])
 print("
Total of Sales % of Total: ", round(( train[train['totals.transactionRevenue'] != np.nan]['totals.transactionRevenue'].count() / len(train['totals.transactionRevenue'])* 100),4))
return
dfTest['text_w'], dfTest['text_w_n'], _ = zip(*dfTest.apply(lambda df: stringtowords(df.text,""), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: cleanstring(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: tweetCleaner(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: cleanstring2(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: correctSpelling(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'] = dfTest.apply(lambda df: addSpace(df.text_w), axis=1)
 dfTest.sample(n=5, random_state=50 )<load_pretrained>
missing_values(train )
print('Loading RoBERTa tokenizer...')
 tokenizer = RobertaTokenizer.from_pretrained(model_dir )<string_transform>
test['totals.transactionRevenue'] = np.nan
def robertaTokenize(oldtext_w, oldtext_w_n = None):
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = tokenizer.tokenize(current_word)
 newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n<train_model>
all_data = train.append(test, sort=False ).reset_index(drop=True )
test_string = [" hello"," worldasd"]
print("Before robertaTokenize")
 print(test_string)
 print("After robertaTokenize")
 print(robertaTokenize(test_string)[0] )<feature_engineering>
null_cnt = train.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: robertaTokenize(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: robertaTokenize(df.selected_w), axis=1))
print("===== E> Tokenized every word using RoBERTa tokenization <3 =====" )<feature_engineering>
for col in ['trafficSource.keyword',
'trafficSource.referralPath',
'trafficSource.adContent']:
all_data[col].fillna('unknown', inplace=True)
 all_data['totals.pageviews'].fillna(1, inplace=True)
 all_data['totals.newVisits'].fillna(0, inplace=True)
 all_data['totals.bounces'].fillna(0, inplace=True)
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int)
 all_data['totals.newVisits'] = all_data['totals.newVisits'].astype(int)
 all_data['totals.bounces'] = all_data['totals.bounces'].astype(int)
 all_data['trafficSource.isTrueDirect'].fillna(False, inplace=True )
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: robertaTokenize(df.text_w,df.text_w_n), axis=1))
dfTest.sample(n=5, random_state=50 )<define_variables>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
def find_sub_list(sub_list,super_list):
sub_list_length=len(sub_list)
 matching_list = []
found_start = 0
for start in range(0,sub_list_length):
for ind in(j for j,e in enumerate(super_list)if e==sub_list[start]):
found_start = ind-start
if sub_list_length < 2:
return [ind-start,ind-start+sub_list_length-1]
else:
for end in range(sub_list_length,start+1,-1):
if super_list[ind:ind+end-start]==sub_list[start:end]:
return [ind-start,ind-start+sub_list_length-1]
return [found_start, found_start+sub_list_length-1]<sort_values>
all_data[all_data['fullVisitorId'] == '7813149961404844386'].sort_values(by='visitNumber')[
['date','visitId','visitNumber','totals.hits','totals.pageviews']].head(20 )
test_string = ["this","is","a","super","list"]
test_selected_string = ["r","list"]
print("super list")
 print(test_string)
 print("sub list")
 print(test_selected_string)
 print("location that find_sub_list found")
 print(find_sub_list(test_selected_string,test_string))<feature_engineering>
print('unique customDimensions count:', train['customDimensions'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'customDimensions' )
dfTrain['selected_start_end'] = dfTrain.apply(lambda df: find_sub_list(df.selected_w,df.text_w), axis=1)
 dfTrain.head()<split>
format_str = '%Y%m%d'
all_data['formated_date'] = all_data['date'].apply(lambda x: datetime.strptime(str(x), format_str))
all_data['_year'] = all_data['formated_date'].apply(lambda x:x.year)
 all_data['_month'] = all_data['formated_date'].apply(lambda x:x.month)
 all_data['_quarterMonth'] = all_data['formated_date'].apply(lambda x:x.day//8)
 all_data['_day'] = all_data['formated_date'].apply(lambda x:x.day)
 all_data['_weekday'] = all_data['formated_date'].apply(lambda x:x.weekday())
 all_data.drop(['date','formated_date'], axis=1, inplace=True )
if in_testing_mode:
dfTrain, dfLasttest = train_test_split(dfTrain, test_size=0.01, random_state=50)
 dfTrain.reset_index(drop=True, inplace=True)
 dfLasttest.reset_index(drop=True, inplace=True )<set_options>
print('train all:', len(train))
print('train unique fullVisitorId:', train['fullVisitorId'].nunique())
 print('train unique visitId:', train['visitId'].nunique())
 print('-' * 30)
 print('test all:', len(test))
print('test unique fullVisitorId:', test['fullVisitorId'].nunique())
 print('test unique visitId:', test['visitId'].nunique())
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("the device we will run on is: ",device)
 if device.type == "cuda":
print("the GPU is: ", torch.cuda.get_device_name(0))<categorify>
print(all_data['visitNumber'].value_counts() [:5])
 print('-' * 30)
 print(all_data['totals.newVisits'].value_counts())
 print('-' * 30)
 print(all_data['totals.bounces'].value_counts() )
test_question = "positive"
test_string = [" hello"," world"]
test_context = robertaTokenize(test_string)[0]
print("question: ", test_question)
 print("context: ", test_context)
 test = tokenizer.encode_plus(
 [test_question],
test_context,
add_special_tokens = True,
max_length = 10,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 print("input_ids: ",test["input_ids"])
 print("attention_mask: ",test["attention_mask"])
 print("token_type_ids: ",test["token_type_ids"])
 print("decoded: ",tokenizer.decode(test["input_ids"][0]))<define_variables>
all_data['_visitStartHour'] = all_data['visitStartTime'].apply(
 lambda x: str(datetime.fromtimestamp(x ).hour))
max_word_count = max(np.max(dfTrain["text_w"].apply(lambda x: len(x))), np.max(dfTest["text_w"].apply(lambda x: len(x))))
max_word_count+=5
print("Max clean word count is {}".format(max_word_count))<categorify>
print('unique browser count:', train['device.browser'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'device.browser', 10)
def convert_text2ids(sentiment, word_list, output, max_word_count, input_ids, attention_masks, token_type_ids, start_positions, end_positions):
encoded_dict = tokenizer.encode_plus(
 [sentiment],
word_list,
add_special_tokens = True,
max_length = max_word_count,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 token_type_ids.append(encoded_dict['token_type_ids'])
 if output is not None:
start_positions.append(output[0])
 end_positions.append(output[1] )<concatenate>
pd.crosstab(all_data['device.deviceCategory'], all_data['device.isMobile'], margins=False)
 all_data['isMobile'] = True
all_data.loc[all_data['device.deviceCategory'] == 'desktop', 'isMobile'] = False
train_input_ids, train_attention_masks, train_token_type_ids, train_start_positions, train_end_positions = [], [], [], [], []
dfTrain.apply(lambda df: convert_text2ids(df.sentiment, df.text_w, df.selected_start_end, max_word_count, train_input_ids, train_attention_masks, train_token_type_ids, train_start_positions, train_end_positions), axis=1)
 train_input_ids = torch.cat(train_input_ids, dim=0)
 train_attention_masks = torch.cat(train_attention_masks, dim=0)
 train_token_type_ids = torch.cat(train_token_type_ids, dim=0)
 train_start_positions = torch.tensor(train_start_positions, dtype = torch.long)
 train_end_positions = torch.tensor(train_end_positions, dtype = torch.long )<set_options>
print('unique operatingSystem count:', train['device.operatingSystem'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'device.operatingSystem', 10 )
batch_size = 64
epochs = 4
learning_rate = 5e-5
adam_epsilon = 1e-8
validation_size = 0.01
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )<split>
print('unique geoNetwork.city count:', train['geoNetwork.city'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.city', 10 )
data = TensorDataset(train_input_ids,
train_attention_masks,
train_token_type_ids,
train_start_positions,
train_end_positions)
 train_data_size = int(( 1-validation_size)* len(data))
val_data_size = len(data)- train_data_size
train_dataset, val_dataset = torch.utils.data.random_split(data, [train_data_size, val_data_size])
 train_dataloader = DataLoader(train_dataset,
sampler=RandomSampler(train_dataset),
batch_size = batch_size)
 val_dataloader = DataLoader(val_dataset,
sampler=SequentialSampler(val_dataset),
batch_size = batch_size)
 print("train data length :", len(train_dataset))
print("validation data length :", len(val_dataset))<define_variables>
print('unique geoNetwork.region count:', train['geoNetwork.region'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.region', 10 )
random_step = random.randint(0,len(train_dataloader))
for step, batch in enumerate(train_dataloader):
if step == random_step:
random_data = random.randint(0,len(batch[0]))
sample_input_id = batch[0][random_data]
sample_attention_mask = batch[1][random_data]
sample_token_type_id = batch[2][random_data]
sample_start_position = batch[3][random_data]
sample_end_position = batch[4][random_data]
break
print("sample_input_id")
 print("\t",sample_input_id)
 print("sample_attention_mask")
 print("\t",sample_attention_mask)
 print("sample_token_type_id")
 print("\t",sample_token_type_id)
 print("sample_start_position")
 print("\t",sample_start_position)
 print("sample_end_position")
 print("\t",sample_end_position )<choose_model_class>
print('unique geoNetwork.continent count:', train['geoNetwork.continent'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'geoNetwork.continent' )
class tweetModel(BertPreTrainedModel):
def __init__(self, conf):
super(tweetModel, self ).__init__(conf)
 self.roberta = RobertaModel.from_pretrained(model_dir, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, dictionary):
outputs = self.roberta(
 dictionary["input_ids"],
attention_mask=dictionary["attention_mask"],
token_type_ids=dictionary["token_type_ids"] 
)
 out = torch.cat(( outputs[2][-1], outputs[2][-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 outputs =(start_logits, end_logits)
 if "start_positions" in dictionary and "end_positions" in dictionary:
if dictionary["start_positions"] is not None and dictionary["end_positions"] is not None:
if len(dictionary["start_positions"].size())> 1:
dictionary["start_positions"] = dictionary["start_positions"].squeeze(-1)
 if len(dictionary["end_positions"].size())> 1:
dictionary["end_positions"] = dictionary["end_positions"].squeeze(-1)
 ignored_index = start_logits.size(1)
 dictionary["start_positions"].clamp_(0, ignored_index)
 dictionary["end_positions"].clamp_(0, ignored_index)
 loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
 start_loss = loss_fct(start_logits, dictionary["start_positions"])
 end_loss = loss_fct(end_logits, dictionary["end_positions"])
 total_loss =(start_loss + end_loss)/ 2
outputs =(total_loss,)+ outputs
return outputs<load_pretrained>
print(all_data['totals.hits'].value_counts() [:10])
 all_data['totals.hits'] = all_data['totals.hits'].astype(int )
rb_config = RobertaConfig.from_pretrained(model_dir , output_hidden_states = True)
 model = tweetModel(conf=rb_config)
 model.to(device )<choose_model_class>
print(all_data['totals.pageviews'].value_counts() [:10])
 all_data['totals.pageviews'] = all_data['totals.pageviews'].astype(int )
param_optimizer = list(model.named_parameters())
 param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]
no_decay =['bias','LayerNorm.weight']
optimizer_grouped_parameters =[
{'params':[ p for n , p in param_optimizer
if not any(nd in n for nd in no_decay)],'weight_decay':0.01},
{'params':[p for n, p in param_optimizer
if any(nd in n for nd in no_decay)],'weigth_decay':0.0}
]
optimizer = AdamW(optimizer_grouped_parameters,
lr = learning_rate,
eps = adam_epsilon
)<train_model>
print('unique trafficSource.adContent count:', train['trafficSource.adContent'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.adContent', 10)
 all_data['_adContentGMC'] =(all_data['trafficSource.adContent'] == 'Google Merchandise Collection' ).astype(np.uint8 )
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<train_model>
print('unique trafficSource.campaign count:', train['trafficSource.campaign'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.campaign', 10)
 all_data['_withCampaign'] =(all_data['trafficSource.campaign'] != '(not set)' ).astype(np.uint8 )
def train(model, optimizer, scheduler, epochs, train_dataloader, validation_dataloader, training_stats):
total_t0 = time.time()
for epoch_i in range(0, epochs):
print('
======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
batch_device = tuple(t.to(device)for t in batch)
 if step % 20 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
inputs = {
"input_ids": batch_device[0],
"attention_mask": batch_device[1],
"token_type_ids": batch_device[2],
"start_positions": batch_device[3],
"end_positions": batch_device[4],
}
model.zero_grad()
outputs = model(inputs)
 loss = outputs[0]
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("
Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(training_time))
print("
Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
batch_device = tuple(t.to(device)for t in batch)
 inputs = {
"input_ids": batch_device[0],
"attention_mask": batch_device[1],
"token_type_ids": batch_device[2],
"start_positions": batch_device[3],
"end_positions": batch_device[4],
}
with torch.no_grad() :
outputs = model(inputs)
 loss = outputs[0]
total_eval_loss += loss.item()
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append({
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Training Time': training_time,
'Validation Time': validation_time
})
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )<train_model>
print('unique trafficSource.keyword count:', train['trafficSource.keyword'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.keyword', 10 )
training_stats = []
train(model, optimizer, scheduler, epochs, train_dataloader, val_dataloader, training_stats )<create_dataframe>
print('unique trafficSource.medium count:', train['trafficSource.medium'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.medium' )
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 df_stats<define_variables>
print('unique trafficSource.source count:', train['trafficSource.source'].nunique())
 plotCategoryRateBar(all_data, train_rev, 'trafficSource.source', 10)
 all_data['_sourceGpmall'] =(all_data['trafficSource.source'] == 'mall.googleplex.com' ).astype(np.uint8 )
random_step = random.randint(0,len(val_dataloader))
for step, batch in enumerate(val_dataloader):
if step == random_step:
random_data = random.randint(0,len(batch[0]))
sample_input_id = batch[0][random_data].to(device)
 sample_attention_mask = batch[1][random_data].to(device)
 sample_token_type_id = batch[2][random_data].to(device)
 sample_start_position = batch[3][random_data].to(device)
 sample_end_position = batch[4][random_data].to(device)
 break
print("sample_input_id")
 print("\t",sample_input_id)
 print("sample_attention_mask")
 print("\t",sample_attention_mask)
 print("sample_token_type_id")
 print("\t",sample_token_type_id)
 test_inputs = {
"input_ids": sample_input_id.view(1,-1 ).to(device),
"attention_mask": sample_attention_mask.view(1,-1 ).to(device),
"token_type_ids": sample_token_type_id.view(1,-1 ).to(device)
 }
sample_answer_start_scores, sample_answer_end_scores = model(test_inputs)
 answer_start = torch.argmax(sample_answer_start_scores)
 answer_end = torch.argmax(sample_answer_end_scores)
 print("
Real start: ", sample_start_position, 'Predicted start: ', answer_start)
 print("Real end: ", sample_end_position, 'Predicted end: ', answer_end )<sort_values>
_=
all_data['_meanHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('mean')
 all_data['_meanHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('mean')
 all_data['_sumHitsPerDay'] = all_data.groupby(['_day'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerWeekday'] = all_data.groupby(['_weekday'])['totals.hits'].transform('sum')
 all_data['_sumHitsPerMonth'] = all_data.groupby(['_month'])['totals.hits'].transform('sum')
 for feature in ['totals.hits', 'totals.pageviews']:
info = all_data.groupby('fullVisitorId')[feature].mean()
all_data['_usermean_' + feature] = all_data.fullVisitorId.map(info)
 for feature in ['visitNumber']:
info = all_data.groupby('fullVisitorId')[feature].max()
all_data['_usermax_' + feature] = all_data.fullVisitorId.map(info)
 del info
def getSelected(text, text_w_n, answer_start_scores, answer_end_scores):
start_sorted_indexes = np.argsort(-answer_start_scores, kind='quicksort', order=None)[0]
end_sorted_indexes = np.argsort(-answer_end_scores, kind='quicksort', order=None)[0]
found_start_end = False
for i in range(0,len(start_sorted_indexes)) :
current_start = start_sorted_indexes[i]
if current_start >= len(text_w_n):
continue
for j in range(0,i+2 if i+2<len(end_sorted_indexes)else len(end_sorted_indexes)) :
current_end = end_sorted_indexes[j]
if current_end >= len(text_w_n):
continue
if current_start <= current_end:
found_start_end = True
real_start = text_w_n[current_start]
real_end = text_w_n[current_end]
break
if found_start_end:
break
if not found_start_end:
return text
splitted_text = text.split()
selected_text = splitted_text[real_start:real_end+1]
selected_text = " ".join(selected_text)
 return selected_text
def predict_test(text, sentiment, text_w, text_w_n):
encoded_dict = tokenizer.encode_plus(
 [sentiment],
text_w,
add_special_tokens = True,
max_length = max_word_count,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 encoded_dict["input_ids"] = encoded_dict["input_ids"].to(device)
 encoded_dict["token_type_ids"] = encoded_dict["token_type_ids"].to(device)
 encoded_dict["attention_mask"] = encoded_dict["attention_mask"].to(device)
 answer_start_scores, answer_end_scores = model(encoded_dict)
 answer_start_scores = answer_start_scores.cpu().detach().numpy()
answer_end_scores = answer_end_scores.cpu().detach().numpy()
selected_text = getSelected(text, text_w_n, answer_start_scores, answer_end_scores)
 return selected_text<predict_on_test>
all_data['_source.country'] = all_data['trafficSource.source'] + '_' + all_data['geoNetwork.country']
all_data['_campaign.medium'] = all_data['trafficSource.campaign'] + '_' + all_data['trafficSource.medium']
all_data['_browser.category'] = all_data['device.browser'] + '_' + all_data['device.deviceCategory']
all_data['_browser.os'] = all_data['device.browser'] + '_' + all_data['device.operatingSystem']
x = random.randint(0,len(dfTest))
while dfTest.iloc[x].sentiment == "neutral":
x = random.randint(0,len(dfTest))
print(dfTest.iloc[x].text)
 print(dfTest.iloc[x].sentiment)
 predict_test(dfTest.iloc[x].text, dfTest.iloc[x].sentiment, dfTest.iloc[x].text_w, dfTest.iloc[x].text_w_n )<feature_engineering>
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )
dfTest["selected_text"] = dfTest.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 dfTest[["textID", "text", "sentiment", "selected_text"]].head(20 )<load_from_csv>
all_data.drop(['visitId','visitStartTime'],axis=1,inplace=True)
 for i, t in all_data.loc[:, all_data.columns != 'fullVisitorId'].dtypes.iteritems() :
if t == object:
all_data[i].fillna('unknown', inplace=True)
 all_data[i] = pd.factorize(all_data[i])[0]
dfContrib = pd.read_csv(sample_submission_file)
 dfContrib.head()<save_to_csv>
train = all_data[all_data['totals.transactionRevenue'].notnull() ]
test = all_data[all_data['totals.transactionRevenue'].isnull() ].drop(['totals.transactionRevenue'], axis=1 )
dfContrib = pd.merge(dfContrib[["textID"]], dfTest[["textID","selected_text"]], on='textID')
 dfContrib.to_csv(submission_file,index=False)
 pd.read_csv(submission_file ).head()<string_transform>
train_id = train['fullVisitorId']
test_id = test['fullVisitorId']
Y_train_reg = train.pop('totals.transactionRevenue')
 X_train = train.drop(['fullVisitorId'], axis=1)
 X_test = test.drop(['fullVisitorId'], axis=1)
 print(X_train.shape, X_test.shape )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<predict_on_test>
del all_data, train, test, train_rev
gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )
dfNeutral = dfTrain[dfTrain['sentiment'] == 'neutral']
dfNeutral['jaccard'] = dfNeutral.apply(lambda df: jaccard(df.selected_text, df.text), axis=1)
 print("Average jaccard score of neutral data when text used as selected text: ",dfNeutral["jaccard"].mean())
 dfNeutral["predicted_selected_text"] = dfNeutral.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfNeutral['predicted_jaccard'] = dfNeutral.apply(lambda df: jaccard(df.selected_text, df.predicted_selected_text), axis=1)
 print("Average jaccard score of neutral data when models prediction used as selected text: ",dfNeutral["predicted_jaccard"].mean())
 del(dfNeutral )<predict_on_test>
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.6,
'feature_fraction': 0.6
}
folds = GroupKFold(n_splits=5)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train_reg, groups=train_id)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train_reg.iloc[trn_]
val_x, val_y = X_train.iloc[val_], Y_train_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=3000)
 reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=50, verbose=500)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 sub_preds += reg.predict(X_test, num_iteration=reg.best_iteration_)/ folds.n_splits
pred = sub_preds
if in_testing_mode:
dfLasttest["predicted_selected_text"] = dfLasttest.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfLasttest["jaccard"] = dfLasttest.apply(lambda df: jaccard(df.selected_text, df.predicted_selected_text), axis=1)
 print("average jaccard score of test data is: ",dfLasttest["jaccard"].mean())
 print("average jaccard score of positive test data is: ",dfLasttest[dfLasttest['sentiment'] == 'positive']["jaccard"].mean())
 print("average jaccard score of negative test data is: ",dfLasttest[dfLasttest['sentiment'] == 'negative']["jaccard"].mean())
 print("average jaccard score of neutral test data is: ",dfLasttest[dfLasttest['sentiment'] == 'neutral']["jaccard"].mean() )<predict_on_test>
submission = pd.DataFrame({'fullVisitorId':test_id, 'PredictedLogRevenue':pred})
 submission["PredictedLogRevenue"] = np.expm1(submission["PredictedLogRevenue"])
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].apply(lambda x : 0.0 if x < 0 else x)
 submission["PredictedLogRevenue"] = submission["PredictedLogRevenue"].fillna(0.0)
 submission_sum = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId' ).sum().reset_index()
submission_sum["PredictedLogRevenue"] = np.log1p(submission_sum["PredictedLogRevenue"])
 submission_sum.to_csv("submission.csv", index=False)
 submission_sum.head(20 )