Kaggle competition: Google Analytics Customer Revenue Prediction
percent_true = 0.42966<load_from_csv>
def load_df(csv_path, nrows=None):
USE_COLUMNS = [
'channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId',
'visitNumber', 'visitStartTime', 'customDimensions'
]
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=USE_COLUMNS)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df
df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 print(df_test.head() )<string_transform>
%%time
train_df = load_df(".. /input/train_v2.csv")
 test_df = load_df(".. /input/test_v2.csv" )
combined_text = ""
combined_true_text = ""
combined_false_text = ""
for index, row in df_train.iterrows() :
row_text = row['text'].lower() + ' '
combined_text += row_text
if row['target']:
combined_true_text += row_text
else:
combined_false_text += row_text
combined_text_words = nltk.word_tokenize(combined_text)
 true_text_words = nltk.word_tokenize(combined_true_text)
 false_text_words = nltk.word_tokenize(combined_false_text)
 print(combined_text_words[:50])
 print(true_text_words[:50])
 print(false_text_words[:50] )<sort_values>
train_df = train_df.drop(labels=['trafficSource.campaignCode'],axis=1)
 train_df = train_df.drop(labels=['customDimensions','totals.totalTransactionRevenue'],axis=1)
 test_df = test_df.drop(labels=['customDimensions','totals.transactionRevenue','totals.totalTransactionRevenue'],axis=1 )
proba_true_list = [(key, freq_dist_true[key]*percent_true*len(combined_text_words)/(len(true_text_words)*freq_dist_combined[key])) for key in freq_dist_true.keys() if len(key)>3 and freq_dist_combined[key]>=25]
proba_true_list = sorted(proba_true_list, key=lambda x: np.absolute(percent_true - x[1]), reverse=True)
 print(proba_true_list )<define_variables>
train_df['date']= pd.to_datetime(train_df['date'],format='%Y%m%d')
 test_df['date']= pd.to_datetime(test_df['date'],format='%Y%m%d')
 train_df.head()
important_word_list = [tpl[0] for tpl in proba_true_list if np.absolute(tpl[1]-percent_true)>0.1]
for w in important_word_list:
print(w )<string_transform>
train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].astype('float')
 print('Total records: ', len(train_df), 'customers doesnt contribute for revenue: ',train_df['totals.transactionRevenue'].isna().sum() , \
'customers contributing revenue: ',len(train_df)- train_df['totals.transactionRevenue'].isna().sum() )
sent_list = [nltk.word_tokenize(row['text'].lower())for(item, row)in df_train.iterrows() ] + [nltk.word_tokenize(row['text'].lower())for(item, row)in df_test.iterrows() ]<choose_model_class>
print('Features with one unique values are :')
 print(train_df.columns[train_df.nunique() == 1])
 new_df = train_df.drop(labels = train_df.columns[train_df.nunique() == 1],
axis = 1)
 test_df = test_df.drop(labels = train_df.columns[train_df.nunique() == 1],
axis = 1 )
word_embedding = Word2Vec(sent_list, size=75, min_count=5, workers=20, window=5,iter=1,sg=1 )<train_model>
geofields = ['geoNetwork.city','geoNetwork.continent',
'geoNetwork.country','geoNetwork.metro',
'geoNetwork.networkDomain','geoNetwork.region',
'geoNetwork.subContinent']
for fields in geofields:
temp_df = new_df.groupby(by=fields ).size().sort_values(ascending=False ).head(10)
 print(temp_df)
 print('*'*30 )
word_embedding.train(alt_corpus, total_examples=word_embedding.corpus_count, epochs=25 )<train_model>
new_df.isna().sum()
word_embedding.train(sent_list, total_examples=word_embedding.corpus_count, epochs=100 )<compute_test_metric>
new_df1 = new_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro',
'geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],
axis=1)
 test_df1 = test_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro',
'geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],
axis=1 )
important_word_list_2 = [important_word_list[0]]
for w in important_word_list[1:]:
add_word = True
for w2 in important_word_list_2:
if np.absolute(word_embedding.wv.similarity(w,w2)) >0.65:
add_word = False
if add_word:
important_word_list_2.append(w)
 for w in important_word_list_2:
print(w )<prepare_output>
new_df1 = new_df1.drop(labels=['date','device.isMobile'], axis=1)
 test_df1 = test_df1.drop(labels=['date','device.isMobile'], axis=1 )
important_word_list = important_word_list_2<compute_test_metric>
new_df1 = new_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId',
'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot',
'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],
axis=1)
 test_df1 = test_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId',
'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot',
'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],
axis=1)
 print(new_df1.shape)
 new_df1.head()
def max_similarity(wd, text):
ret = 0
words = nltk.word_tokenize(text.lower())
 for w in words:
x = 0
if w in word_embedding.wv.vocab:
x = word_embedding.wv.similarity(w,wd)
 if x>ret:
ret=x
return ret<categorify>
new_df1.fillna('0',inplace=True)
 test_df1.fillna('0',inplace=True )
def contains_one(s,lst):
if not isinstance(s,str):
s = s.decode('utf-8')
 for w in lst:
if re.search(re.escape(w),s,re.IGNORECASE):
return 1
return 0<compute_test_metric>
new_df1.isna().sum()
for w in important_word_list:
df_train[w] = df_train.apply(lambda x: max_similarity(w, x['text']), axis=1)
 df_test[w] = df_test.apply(lambda x: max_similarity(w, x['text']), axis=1 )<create_dataframe>
new_df1['trafficSource.adContent'].fillna('NoAdContent',inplace=True)
 new_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 test_df1['trafficSource.adContent'].fillna('NoAdContent',inplace=True)
 test_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 for columns in ['totals.sessionQualityDim','totals.timeOnSite',
'totals.transactions']:
new_df1[columns].fillna('0',inplace=True)
 new_df1[columns] = new_df1[columns].astype('int')
 test_df1[columns].fillna('0',inplace=True)
 test_df1[columns] = test_df1[columns].astype('int' )
df_targets = df_train['target']
df_prediction = pd.DataFrame()
df_prediction['id'] = df_test['id']<choose_model_class>
def convert_category_todummies(df,field):
dummy_df = pd.get_dummies(df[field])
 df = pd.concat([df,dummy_df],axis=1)
 df.drop(labels=[field],axis=1,inplace=True)
 return df
model = SVC()<train_model>
def convert_category_tolevel(df,field):
df[field],index = pd.factorize(df[field])
 return df
model.fit(df_train, df_targets )<predict_on_test>
merged_df['totals.transactionRevenue'] = np.log1p(merged_df['totals.transactionRevenue'])
 new_df1 = merged_df.iloc[:train_size]
test_df1 = merged_df[train_size:]
test_df1 = test_df1.drop(labels=['totals.transactionRevenue'],axis=1)
 print(new_df1.shape, test_df1.shape )
predictions = model.predict(df_test )<feature_engineering>
del train_df, new_df, test_df,merged_df, transaction_df, nontransaction_df, plot_df
gc.collect()
df_prediction['target'] = predictions<save_to_csv>
train_x = new_df1.groupby(by=['fullVisitorId'] ).mean()
del new_df1
gc.collect()
df_prediction.to_csv("predictions.csv", index=False )<load_from_csv>
scaler1 = MinMaxScaler()
train_y = train_x['totals.transactionRevenue']
train_x = train_x.drop(labels=['totals.transactionRevenue'],axis=1)
 scaled_x = scaler1.fit_transform(train_x.values)
 train_x = pd.DataFrame(scaled_x, columns=train_x.columns )
train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print(train_data.shape)
 train_data.head()<count_missing_values>
X_train, X_val, y_train, y_val = train_test_split(train_x, train_y, test_size=0.15, random_state=1,)
 lgb_train_data = lgb.Dataset(X_train, label=y_train)
 lgb_val_data = lgb.Dataset(X_val, label=y_val)
 params = {
"objective" : "regression",
"metric" : "rmse",
"max_depth" : 10,
"boosting" : "gbdt",
"learning_rate" : 0.0025,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"seed": 42}
lgbModel = lgb.train(params, lgb_train_data,
num_boost_round=5000,
valid_sets=[lgb_train_data, lgb_val_data],
early_stopping_rounds=100,
verbose_eval=500 )
train_data.isnull().sum()<correct_missing_values>
test_x = test_df1.groupby(by=['fullVisitorId'] ).mean()
visitorid = test_x.index
scaled_test = scaler1.transform(test_x)
 test_x = pd.DataFrame(scaled_test,columns=test_x.columns )
train_data.fillna(value = 'None', inplace = True )<load_from_csv>
y_pred_train = lgbModel.predict(X_train, num_iteration=model.best_iteration)
 y_pred_val = lgbModel.predict(X_val, num_iteration=model.best_iteration)
 y_pred_submit = lgbModel.predict(test_x, num_iteration=model.best_iteration)
 print(f"LGBM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )
sample_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print(sample_submission.shape)
 sample_submission.head()<set_options>
if torch.cuda.is_available() :
device = torch.device('cuda')
 else:
device = torch.device('cpu')
 print(device )<load_pretrained>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case = True )<define_variables>
submission = pd.DataFrame({'fullVisitorId':visitorid,'PredictedLogRevenue':np.squeeze(y_pred_submit)})
 submission['fullVisitorId']= submission['fullVisitorId'].astype(str)
 submission['PredictedLogRevenue']=submission['PredictedLogRevenue'].apply(lambda x: 0 if x<0 else x)
 submission.to_csv('submission.csv',index=False )