Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<categorify>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<set_options>
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None )
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )<train_model>
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
seed = 50
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 loss_arr = []
for i in range(0, epochs):
print('====== Epoch {:} of {:}'.format(i+1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 30 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print('Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids = None,
attention_mask = b_input_mask,
labels = b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_arr.append(avg_train_loss)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in val_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids = None,
attention_mask = b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<define_variables>
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['number_of_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 df['ratio_pageviews'] = df['totals.pageviews'] / df['number_of_pageviews']
df['number_of_sessions'] = df['date'].map(
 df[['date']].groupby('date' ).size() 
)
 y_reg = train['target']
del train['target']
pred_labels = np.array(sample_submission['target'] )<feature_engineering>
mobile_browsers = ['amazon', 'android', 'blackberry', 'browser', 'chrome', 'in-app', 'iphone', 'konqueror', 'lunascape',
'mini', 'mozilla', 'netscape', 'nokia', 'playstation', 'puffin', 'samsung']
browsers = ['chrome', 'coc coc', 'edge', 'firefox', 'internet explorer', 'iron', 'maxthon', 'opera', 'safari']
key_sources = ['ad', 'amazon', 'arstechnica', 'ask', 'baidu', 'baidu', 'bing', 'businessinsider', 'dealspotr',
'dfa', 'direct', 'duckduckgo', 'edu', 'facebook', 'feedly', 'flipboard', 'github', 'golang', 'google',
'gophergala', 'linkedin', 'lunametrics', 'mail', 'mdn', 'messenger', 'metrics', 'msn', 'mysearch',
'outlook', 'partners', 'phandroid', 'pinterest', 'qiita', 'reddit', 'sashihara', 'search', 'seroundtable',
'siliconvalley', 'sm.cn', 'sogou', 'squishable', 't-online.de', 't.co', 'twitter', 'vk.com', 'wow',
'yahoo', 'yandex', 'youtube']
def browser_mapping(x):
if x in browsers:
return x
elif any([word in x for word in mobile_browsers]):
return 'mobile_browser'
elif '(not set)' in x:
return 'nan'
else:
return 'others'
def adcontents_mapping(x):
if 'google' in x:
return 'google'
elif '(not set)' in x or 'nan' in x:
return 'nan'
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
for word in key_sources:
if word in x:
return word
if '(not set)' in x or 'nan' in x:
return 'nan'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )
te_labels = torch.tensor(pred_labels )<create_dataframe>
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
prediction_data = TensorDataset(te_input, te_mask, te_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE )<predict_on_test>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )
print('Predicting labels for {:,} test sentences...'.format(len(te_input)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 true_labels.append(label_ids)
 print(' DONE.' )<concatenate>
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
sample_submission['target'] = flat_predictions
sample_submission.to_csv('submission.csv', index = False )<load_from_csv>
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
plt.rcParams.update({'font.size': 14})
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 print(train.shape, test.shape, sub_sample.shape )<save_to_csv>
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
leak = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv", encoding='latin_1')
 leak['target'] =(leak['choose_one']=='Relevant' ).astype(int)
 leak['id'] = leak.index
leak = leak[['id', 'target','text']]
merged_df = pd.merge(test, leak, on='id')
 sub1 = merged_df[['id', 'target']]
sub1.to_csv('submit.csv', index=False )<set_options>
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 all_data = [train,test]<groupby>
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape
print(train[['Pclass', 'Sex']].groupby(['Sex'], as_index=False ).mean() )<groupby>
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean() )<define_variables>
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds1 = np.zeros(full_data.shape[0])
 oof_preds2 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("-"* 20 + "Fold :"+str(fold_)+ "-"* 20)
 trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg1 = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 print("-"* 20 + "LightGBM_1 Training" + "-"* 20)
 reg1.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 print("-"* 20 + "LightGBM_2 Training" + "-"* 20)
 reg2 = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.8,
colsample_bytree=.8,
random_state=3 
)
 reg2.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 oof_preds1[val_] = reg1.predict(val_x, num_iteration=reg1.best_iteration_)
 oof_preds1[oof_preds1 < 0] = 0
oof_preds2[val_] = reg2.predict(val_x, num_iteration=reg2.best_iteration_)
 oof_preds2[oof_preds2 < 0] = 0
both_oof[val_] = 0.5 * oof_preds1[val_] + 0.5 * oof_preds2[val_]
_preds1 = reg1.predict(sub_full_data[full_data.columns], num_iteration=reg1.best_iteration_)
 _preds1[_preds1 < 0] = 0
_preds2 = reg2.predict(sub_full_data[full_data.columns], num_iteration=reg2.best_iteration_)
 _preds2[_preds2 < 0] = 0
sub_preds +=(_preds1 / len(folds)) * 0.4 +(_preds2 / len(folds)) * 0.5
print("LGB_1 ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1)**.5)
 print("LGB_2 ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds2)**.5)
 print("Combined ", mean_squared_error(np.log1p(trn_user_target['target']), both_oof)**.5 )
women = train.loc[train.Sex == 'female']['Survived']
rate_women = sum(women)/len(women)
 print('The rate of survival for women is:', rate_women )<define_variables>
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )