Kaggle competition: Google Analytics Customer Revenue Prediction
df_mislabeled = df_train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled.index.tolist()<feature_engineering>

 def drop_messy_cols(df):
numeric_cols=list(df.select_dtypes(include=['int32','int64','float32'] ).columns)
 categorical_cols=list(df.select_dtypes(include=['object'] ).columns)
 bool_cols=list(df.select_dtypes(include=['bool'] ).columns)

low_cardinality_cols=[col for col in categorical_cols if df[col].nunique() <10]
print('
',df[low_cardinality_cols].nunique())

high_cardinality_cols=[col for col in categorical_cols if df[col].nunique() >10 and col not in [
'totals.transactionRevenue','fullVisitorId', 'totals.hits', 'device.operatingSystem', 'geoNetwork.metro'
'totals.pageviews','trafficSource.source']
]
print('
',df[high_cardinality_cols].nunique())

high_na_cols=[col for col in df.columns if df[col].isna().sum() >10000 and col not in ['totals.transactionRevenue']]
print('
',df[high_na_cols].isna().sum())
 _COLS_TO_DROP = ['visitNumber', 'totals.newVisits', 'totals.bounces', 'trafficSource.isTrueDirect', 'geoNetwork.city', 'geoNetwork.networkDomain',
'geoNetwork.region', 'geoNetwork.subContinent', 'trafficSource.adContent', 'trafficSource.campaign', 'trafficSource.keyword',
'trafficSource.referralPath', 'device.isMobile'
]
df.drop(columns=_COLS_TO_DROP , inplace=True, axis=1)
 return df
def preprocess_and_impute(train_df, test_df):
full_df=pd.concat([train_df,test_df],axis=0)

full_df=drop_messy_cols(full_df)

_CNTRYS_LT_4_TRANSACTIONS = list(full_df.groupby('geoNetwork.country')['totals.transactions'].sum() [full_df.groupby('geoNetwork.country')['totals.transactions'].sum() <4].index)
 full_df['geoNetwork.country'][full_df['geoNetwork.country'].isin(_CNTRYS_LT_4_TRANSACTIONS)]='Other'

 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'][~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'

 _VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'][~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'

 _SPARSE_METROS= list(full_df.groupby('geoNetwork.metro')['totals.transactions'].sum() [full_df.groupby('geoNetwork.metro')['totals.transactions'].sum() <1].index)
 print('
Found {} sparse-metros.The entire dataset still has {} metros.'.format(len(set(_SPARSE_METROS)) , len(set(full_df['geoNetwork.metro'])) -len(set(_SPARSE_METROS))))
full_df['geoNetwork.metro'][full_df['geoNetwork.metro'].isin(_SPARSE_METROS)]='Other'

 _INEFFECTIVE_TRAFFIC_SOURCES=list(full_df.groupby('trafficSource.source')['totals.transactions'].sum() [full_df.groupby('trafficSource.source')['totals.transactions'].sum() <1].index)
 print('
Found {} ineffective traffic-sources.The entire dataset still has {} traffic-sources.'.format(len(set(_INEFFECTIVE_TRAFFIC_SOURCES)) , len(set(full_df['trafficSource.source'])) -len(set(_INEFFECTIVE_TRAFFIC_SOURCES))))
full_df['trafficSource.source'][full_df['trafficSource.source'].isin(_INEFFECTIVE_TRAFFIC_SOURCES)]='Other'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('google')]='google'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('facebook')]='facebook'
full_df['trafficSource.source'][full_df['trafficSource.source'].str.contains('yahoo')]='yahoo'
print('
LIST OF trafficSource.source: ',list(full_df['trafficSource.source'].value_counts().index))
full_df['date.year']=full_df['date'].dt.year
full_df['date.month']=full_df['date'].dt.month
full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
del(full_df['date'])

_CAT_COLS_FOR_ENCODER = ['channelGrouping', 'date.year','date.month', 'device.browser', 'device.deviceCategory', 'device.operatingSystem',
'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'trafficSource.medium', 'trafficSource.source']
for col in _CAT_COLS_FOR_ENCODER:
lb = LabelEncoder()
full_df[col]=lb.fit_transform(full_df[col])
 return full_df
df_train['target_relabeled'] = df_train['target'].copy()
df_train.loc[df_train['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == 'Hellfire is surrounded by desires so be careful and donÛªt let your desires control you!
df_train.loc[df_train['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0
df_train.loc[df_train['text'] == '.POTUS
df_train.loc[df_train['text'] == 'CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == '
df_train.loc[df_train['text'] == 'In
df_train.loc[df_train['text'] == 'Who is bringing the tornadoes and floods.Who is bringing the climate change.God is after America He is plaguing her
df_train.loc[df_train['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B.Cooper.http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1
df_train.loc[df_train['text'] == "Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "wowo--=== 12000 Nigerian refugees repatriated from Cameroon", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "He came to a land which was engulfed in tribal war and turned it into a land of peace i.e.Madinah.
 df_train.loc[df_train['text'] == "Hellfire! We donÛªt even want to think about it or mention it so letÛªs not do anything that leads to it
df_train.loc[df_train['text'] == "The Prophet(peace be upon him)said 'Save yourself from Hellfire even if it is by giving half a date in charity.'", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "Caution: breathing may be hazardous to your health.", 'target_relabeled'] = 1
df_train.loc[df_train['text'] == "I Pledge Allegiance To The P.O.P.E.And The Burning Buildings of Epic City.??????", 'target_relabeled'] = 0
df_train.loc[df_train['text'] == "
df_train.loc[df_train['text'] == "that horrible sinking feeling when youÛªve been at home on your phone for a while and you realise its been on 3G this whole time", 'target_relabeled'] = 0<count_unique_values>
def load_df_without_hits(path=dirname+'/train_v2.csv'):
ans=pd.DataFrame()
dfs=pd.read_csv(path, parse_dates=['date'],
converters={ j: json.loads for j in _JSON_COLUMNS },
dtype={'fullVisitorId':str}, chunksize=100000 
)
 for df in dfs:
df.reset_index(drop=True, inplace=True)

for col in _JSON_COLUMNS:
column_as_df= pd.json_normalize(df[col])
 column_as_df.columns=[f'{col}.{subcolumn}' for subcolumn in column_as_df.columns]
df=df.drop(col, axis=1 ).merge(column_as_df, right_index=True, left_index=True)

df=df.replace(_COL_VALUES_TO_BE_REPLACED , 'NA')

for col in _COLS_TO_CONVERT_TO_INT:
df[col] = df[col].fillna(1 ).astype(int)

df['totals.transactionRevenue'] = df['totals.transactionRevenue'].fillna(0 ).astype(float)
 df['totals.totalTransactionRevenue'] = df['totals.totalTransactionRevenue'].fillna(0 ).astype(float)

use_df = df[_FEATURES_WITHOUT_HITS]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis = 0 ).reset_index(drop = True)
 if ans.shape[0]%200000==0:
print(f"Loaded {os.path.basename(path)} and now df= {ans.shape}")
 return ans

 _JSON_COLUMNS = [
'device','geoNetwork','totals','trafficSource'
]

 _FEATURES_WITHOUT_HITS = [
'channelGrouping', 'date', 'fullVisitorId', 'socialEngagementType', 'visitId', 'visitNumber', 'visitStartTime',
'device.browser', 'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',
'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region', 'geoNetwork.subContinent',
'totals.hits', 'totals.pageviews', 'totals.transactionRevenue','totals.totalTransactionRevenue', 'totals.sessionQualityDim', 'totals.transactions', 'totals.bounces', 'totals.newVisits', 'totals.timeOnSite',
'trafficSource.medium', 'trafficSource.source', 'trafficSource.isTrueDirect', 'trafficSource.keyword','trafficSource.campaign','trafficSource.referralPath', 'trafficSource.adContent',
'trafficSource.adwordsClickInfo.slot'
]

 _COLS_TO_CONVERT_TO_INT = [
'totals.pageviews', 'totals.hits', 'totals.sessionQualityDim',
'totals.visits', 'totals.timeOnSite', 'totals.transactions'
]

 _COL_VALUES_TO_BE_REPLACED = {
'device.operatingSystem' : ['not available in demo dataset','(not set)'],
'device.browser' : ['not available in demo dataset','(not set)','0'],
'geoNetwork.city' : ['not available in demo dataset','(not set)'],
'geoNetwork.metro' : ['not available in demo dataset', '(not set)'],
'geoNetwork.networkDomain' : ['unknown.unknown', '(not set)', 'not.configured'],
'geoNetwork.region' : ['not available in demo dataset','(not set)'],
'geoNetwork.continent' : ['not available in demo dataset','(not set)'],
'geoNetwork.subContinent' : ['not available in demo dataset','(not set)'],
'geoNetwork.country' : ['not available in demo dataset','(not set)'],
'trafficSource.campaign' : ['not available in demo dataset','(not set)'],
'trafficSource.medium' : ['(none)','(not set)'],
'trafficSource.source' : '(not set)'
}
K = 2
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
 DISASTER = df_train['target'] == 1
print('Whole Training Set Shape = {}'.format(df_train.shape))
print('Whole Training Set Unique keyword Count = {}'.format(df_train['keyword'].nunique()))
print('Whole Training Set Target Rate(Disaster){}/{}(Not Disaster)'.format(df_train[DISASTER]['target_relabeled'].count() , df_train[~DISASTER]['target_relabeled'].count()))
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train['text_cleaned'], df_train['target']), 1):
print('
Fold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, df_train.loc[trn_idx, 'text_cleaned'].shape, df_train.loc[val_idx, 'text_cleaned'].shape))
print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, df_train.loc[trn_idx, 'keyword'].nunique() , df_train.loc[val_idx, 'keyword'].nunique()))<prepare_x_and_y>
%%time
print('{} Reading the 27 Gig 'train_v2.csv' will take ~18 minutes time! Grab a coffee in the meanwhile.{}
'.format('
train = load_df_without_hits(path=dirname+'/train_v2.csv')
 print(list(train.columns))
train_bkp=train.copy()
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.y_train = train_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val, self.y_val = validation_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self, epoch, logs={}):
train_predictions = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.y_train, train_predictions, average='macro')
 train_recall = recall_score(self.y_train, train_predictions, average='macro')
 train_f1 = f1_score(self.y_train, train_predictions, average='macro')
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_predictions = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.y_val, val_predictions, average='macro')
 val_recall = recall_score(self.y_val, val_predictions, average='macro')
 val_f1 = f1_score(self.y_val, val_predictions, average='macro')
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print('
Epoch: {} - Training Precision: {:.6} - Training Recall: {:.6} - Training F1: {:.6}'.format(epoch + 1, train_precision, train_recall, train_f1))
print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))<choose_model_class>
%%time
print('{} Reading the 7 Gig 'test_v2.csv' will take ~4 minutes time! Go hydrate in the meanwhile :){}
'.format('
test = load_df_without_hits(path=dirname+'/test_v2.csv')
 print(list(test.columns))
%%time
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )<choose_model_class>
full_df=pd.concat([train,test],axis=0)
 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'].loc[~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'
_VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'].loc[~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'
full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
full_df['date.month']=full_df['date'].dt.month
full_df['date.year']=full_df['date'].dt.year
print(full_df.nunique() [full_df.nunique() ==1].index)
 del(full_df['totals.bounces'])
 del(full_df['totals.newVisits'])
 del(full_df['socialEngagementType'])
 del(full_df['trafficSource.isTrueDirect'] )
class DisasterDetector:
def __init__(self, bert_layer, max_seq_length=128, lr=0.0001, epochs=15, batch_size=32):
self.bert_layer = bert_layer
self.max_seq_length = max_seq_length
vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 self.lr = lr
self.epochs = epochs
self.batch_size = batch_size
self.models = []
self.scores = {}
def encode(self, texts):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = self.tokenizer.tokenize(text)
 text = text[:self.max_seq_length - 2]
input_sequence = ['[CLS]'] + text + ['[SEP]']
pad_len = self.max_seq_length - len(input_sequence)
 tokens = self.tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * self.max_seq_length
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(self):
input_word_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='segment_ids')
 pooled_output, sequence_output = self.bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 optimizer = SGD(learning_rate=self.lr, momentum=0.8)
 model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
 return model
def train(self, X):
for fold,(trn_idx, val_idx)in enumerate(skf.split(X['text_cleaned'], X['keyword'])) :
print('
Fold {}
'.format(fold))
X_trn_encoded = self.encode(X.loc[trn_idx, 'text_cleaned'].str.lower())
 y_trn = X.loc[trn_idx, 'target_relabeled']
X_val_encoded = self.encode(X.loc[val_idx, 'text_cleaned'].str.lower())
 y_val = X.loc[val_idx, 'target_relabeled']
metrics = ClassificationReport(train_data=(X_trn_encoded, y_trn), validation_data=(X_val_encoded, y_val))
model = self.build_model()
model.fit(X_trn_encoded, y_trn, validation_data=(X_val_encoded, y_val), callbacks=[metrics], epochs=self.epochs, batch_size=self.batch_size)
 self.models.append(model)
 self.scores[fold] = {
'train': {
'precision': metrics.train_precision_scores,
'recall': metrics.train_recall_scores,
'f1': metrics.train_f1_scores
},
'validation': {
'precision': metrics.val_precision_scores,
'recall': metrics.val_recall_scores,
'f1': metrics.val_f1_scores
}
}
def plot_learning_curve(self):
fig, axes = plt.subplots(nrows=K, ncols=2, figsize=(20, K * 6), dpi=100)
 for i in range(K):
sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[i].history.history['val_accuracy'], ax=axes[i][0], label='val_accuracy')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['precision'], ax=axes[i][0], label='val_precision')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['recall'], ax=axes[i][0], label='val_recall')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['f1'], ax=axes[i][0], label='val_f1')
 axes[i][0].legend()
axes[i][0].set_title('Fold {} Validation Classification Report'.format(i), fontsize=14)
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['loss'], ax=axes[i][1], label='train_loss')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['val_loss'], ax=axes[i][1], label='val_loss')
 axes[i][1].legend()
axes[i][1].set_title('Fold {} Train / Validation Loss'.format(i), fontsize=14)
 for j in range(2):
axes[i][j].set_xlabel('Epoch', size=12)
 axes[i][j].tick_params(axis='x', labelsize=12)
 axes[i][j].tick_params(axis='y', labelsize=12)
 plt.show()
def predict(self, X):
X_test_encoded = self.encode(X['text_cleaned'].str.lower())
 y_pred = np.zeros(( X_test_encoded[0].shape[0], 1))
for model in self.models:
y_pred += model.predict(X_test_encoded)/ len(self.models)
 return y_pred<train_on_grid>
full_df[['fullVisitorId','channelGrouping','date','visitNumber','totals.transactions','totals.transactionRevenue']][
full_df['fullVisitorId']=='9974232250427988367'].sort_values(by=['date','visitNumber'] )
clf = DisasterDetector(bert_layer, max_seq_length=128, lr=0.0001, epochs=10, batch_size=32)
 clf.train(df_train )<save_to_csv>
visitors_in_both_years= set(full_df['fullVisitorId'][full_df['date.year']==2016] ).intersection(set(full_df['fullVisitorId'][full_df['date.year']==2017]))
tmp_df=full_df[['fullVisitorId', 'totals.transactionRevenue']][full_df['fullVisitorId'].isin(visitors_in_both_years)]
print(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].count() [(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].count() >1)
 &(tmp_df.groupby('fullVisitorId')['totals.transactionRevenue'].sum() !=0)])
 del(tmp_df)
 del(visitors_in_both_years)
 del(full_df['visitId'] )
y_pred = clf.predict(df_test)
 model_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = np.round(y_pred ).astype('int')
 model_submission.to_csv('model_submission.csv', index=False)
 model_submission.describe()<load_from_csv>
def standardize_traffic_source(val):
if 'google' in val:
return 'google'
elif 'youtube' in val:
return 'youtube'
elif 'facebook' in val:
return 'facebook'
elif 'yahoo' in val:
return 'yahoo'
elif 'baidu' in val:
return 'baidu'
elif 'aol' in val:
return 'aol'
elif 'quora' in val:
return 'quora'
elif 'reddit' in val:
return 'reddit'
elif 'pinterest' in val:
return 'pinterest'
elif 'bing' in val:
return 'bing'
elif 'wow' in val:
return 'wow'
elif 'blackboard' in val:
return 'blackboard'
elif len(val)>20:
return 'Other'
else:
return val
full_df['trafficSource.source']=full_df['trafficSource.source'].apply(standardize_traffic_source)
 categorical_variables=[
'trafficSource.medium','trafficSource.source']
fig, ax = plt.subplots(2,1, figsize=(16, 12))
plt.subplots_adjust(hspace=.5)
 for variable, subplot in zip(categorical_variables, ax.flatten()):
tmp=pd.pivot_table(data=full_df, index=variable, values='totals.transactionRevenue',
aggfunc='mean' ).reset_index().sort_values(by='totals.transactionRevenue', ascending=False)
 sns.barplot(data=tmp.head(50), x=variable, y='totals.transactionRevenue', ax=subplot ).set_xticklabels(subplot.get_xticklabels() ,rotation=90)
 subplot.set_title('Revenue vs {}'.format(variable.replace('trafficSource.','' ).capitalize()), fontdict={'fontweight':'bold'})
 del(tmp)
 plt.show()
df_leak = pd.read_csv('.. /input/disastertweets/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
perfect_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 perfect_submission['target'] = df_test['target'].values
perfect_submission.to_csv('perfect_submission.csv', index=False)
 perfect_submission.describe()<load_pretrained>
def standardize_traffic_keyword(keyword):
keyword=str(keyword ).lower()
if 'goo' in keyword or 'stor' in keyword:
return 'google'
elif 'yo' in keyword or 'yuotub' in keyword or 'yuo tub' in keyword or 'tub' in keyword or 'yutob' in keyword:
return 'youtube'
elif 'tshirt' in keyword or 't shirt' in keyword or 't-shirt' in keyword:
return 'tshirt'
elif 'shirts' in keyword or 'shirt' in keyword:
return 'shirts'
elif 'lamp' in keyword or 'lamps' in keyword:
return 'lamps'
elif 'android' in keyword:
return 'android'
elif 'merchandi' in keyword:
return 'merchandise'
elif bool(re.search(r'\d', keyword)) :
return 'other'
elif 'apparel' in keyword:
return 'apparel'
elif len(keyword)>28:
return 'other'
else:
return keyword
full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].fillna('other')
 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].apply(standardize_traffic_keyword)
 plt.figure(figsize=(24,6))
keywordwise_revenue=pd.pivot_table(data=full_df, index=['trafficSource.keyword'], values='totals.transactionRevenue',
aggfunc='mean' ).sort_values(by='totals.transactionRevenue', ascending=False ).reset_index()
sns.barplot(data=keywordwise_revenue.head(50), x='trafficSource.keyword', y='totals.transactionRevenue', palette='Greens_r')
 plt.title('Revenue vs {}'.format('Keyword'), fontdict={'fontweight':'bold'})
 plt.xticks(rotation=90)
 plt.show()
del(keywordwise_revenue)
TRAIN_FEATURES = ['id', 'keyword', 'location', 'text', 'target', 'text_cleaned', 'target_relabeled']
TEST_FEATURES = ['id', 'keyword', 'location', 'text', 'target', 'text_cleaned']
df_train[TRAIN_FEATURES].to_pickle('train.pkl')
 df_test[TEST_FEATURES].to_pickle('test.pkl')
 print('Training Set Shape = {}'.format(df_train[TRAIN_FEATURES].shape))
print('Training Set Memory Usage = {:.2f} MB'.format(df_train[TRAIN_FEATURES].memory_usage().sum() / 1024**2))
print('Test Set Shape = {}'.format(df_test[TEST_FEATURES].shape))
print('Test Set Memory Usage = {:.2f} MB'.format(df_test[TEST_FEATURES].memory_usage().sum() / 1024**2))<load_from_csv>
def preprocess_and_standardize(train_df, test_df):
full_df=pd.concat([train_df,test_df],axis=0)

full_df['date.weekday']=full_df['date'].dt.weekday
full_df['date.day']=full_df['date'].dt.day
full_df['date.month']=full_df['date'].dt.month
full_df['date.year']=full_df['date'].dt.year

 print('Columns with cardinality=1 ie Constant value : ',full_df.nunique() [full_df.nunique() ==1].index)
 _COLS_TO_DROP=[ 'trafficSource.adContent', 'visitId', 'device.isMobile', 'trafficSource.referralPath',
'totals.totalTransactionRevenue', 'visitNumber', 'visitStartTime',
'totals.bounces','totals.newVisits','socialEngagementType','trafficSource.isTrueDirect',]
for c in _COLS_TO_DROP:
try:
del(full_df[c])
 except:
pass
print('{} I)DROPPED UNNECESSARY COLUMNS {}'.format('

 _VALID_BROWSERS = [ 'Chrome', 'Safari', 'Firefox', 'Internet Explorer', 'Android Webview', 'Edge', 'Samsung Internet']
full_df['device.browser'].loc[~full_df['device.browser'].isin(_VALID_BROWSERS)]='Other'

 _VALID_OPERATING_SYSTEMS = ['Windows', 'Macintosh', 'Android', 'iOS', 'Linux', 'Chrome OS', 'Windows Phone', 'Samsung']
full_df['device.operatingSystem'].loc[~full_df['device.operatingSystem'].isin(_VALID_OPERATING_SYSTEMS)]='Other'

 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].fillna('other')
 full_df['trafficSource.keyword']=full_df['trafficSource.keyword'].apply(standardize_traffic_keyword)
 full_df['trafficSource.source']=full_df['trafficSource.source'].apply(standardize_traffic_source)
 full_df['trafficSource.adwordsClickInfo.slot']=full_df['trafficSource.adwordsClickInfo.slot'].fillna('No Traffic-ad Slot')
 print('{} II)STANDARDIZED COLUMN VALUES {}'.format('

 _CAT_COLS_FOR_ENCODER = ['channelGrouping', 'device.browser', 'device.deviceCategory', 'device.operatingSystem',
'geoNetwork.city','geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro', 'geoNetwork.region',
'geoNetwork.subContinent','geoNetwork.networkDomain','trafficSource.medium', 'trafficSource.source',
'trafficSource.campaign','trafficSource.keyword','trafficSource.adwordsClickInfo.slot']
print('{} III)ENCODING RELEVANT COLUMNS TO LABELS {}'.format('
for col in _CAT_COLS_FOR_ENCODER:
print('Converting <',col,'> to labels.')
 if full_df[col].dtype.name=='object':
lb = LabelEncoder()
full_df[col]=lb.fit_transform(full_df[col])
 return full_df
def return_aggregated_df(input_df, i=0, create_test=False, verbose=True):
if create_test==False:

 feature_df = input_df[(pd.to_datetime(input_df['date'])> time_from.index[i])&(pd.to_datetime(input_df['date'])<= time_end.index[i])].copy()
else:

 feature_df = input_df
if verbose==True:
print('{} A)CREATED BASE-FEATURE DATAFRAME {}'.format('
_COLS_TO_MEDIAN_VAL = ['channelGrouping' , 'device.browser' , 'device.deviceCategory' , 'device.operatingSystem' , 'geoNetwork.city' , 'geoNetwork.country' ,
'geoNetwork.metro' , 'geoNetwork.networkDomain' , 'geoNetwork.region' , 'geoNetwork.subContinent', 'geoNetwork.continent' ,
'trafficSource.medium' , 'trafficSource.source' , 'trafficSource.campaign' , 'trafficSource.keyword' , 'trafficSource.adwordsClickInfo.slot']
_COLS_TO_SUM_VAL = ['totals.hits' , 'totals.sessionQualityDim' , 'totals.transactions', 'totals.pageviews' ]
_COLS_TO_MEAN_VAL = ['date.year', 'date.month', 'date.day', 'date.weekday']
_COLS_TO_STD_DEV_VAL = ['totals.hits' , 'totals.pageviews' , 'totals.sessionQualityDim']
ans=pd.concat([
feature_df.groupby('fullVisitorId')[_COLS_TO_MEDIAN_VAL].median().add_suffix('_median'),
feature_df.groupby('fullVisitorId')[_COLS_TO_MEAN_VAL].mean().add_suffix('_mean'),
feature_df.groupby('fullVisitorId')[_COLS_TO_SUM_VAL].sum().add_suffix('_sum'),
feature_df.groupby('fullVisitorId')[_COLS_TO_STD_DEV_VAL].std(ddof=0 ).add_suffix('_stdDev'),
feature_df.groupby('fullVisitorId')['date'].apply(lambda x:(x.max() -x.min())//10**9 ).rename('time_diff')
 ], axis=1 ).reset_index()
if verbose==True:
print('{} B)AGGREGATED THE BASE FEATURES {}'.format('
if create_test==False:

 target_df = input_df[(input_df['date'] > pd.Timestamp(time_from.values[i]))
&(input_df['date'] <= pd.Timestamp(time_from.values[i+1])) ].copy()
ans=ans.merge(target_df.groupby('fullVisitorId')['totals.transactionRevenue'].sum().reset_index() , how = 'left', on = 'fullVisitorId')
 if verbose==True:
print('{} C)MERGED THE FUTURE TIMEFRAME TARGET WITH AGGREGATED FEATURES {}'.format('
ans['totals.transactionRevenue']=ans['totals.transactionRevenue'].fillna(0 ).apply(lambda y: np.log(y)if y>0 else y)
 return ans
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 print(df_train.head() )<count_values>
submissions_df=pd.DataFrame()
test_df = full_df.iloc[train_shape[0]:, :]
test_x=return_aggregated_df(test_df,i=0,create_test=True)
 submissions_df['fullVisitorId']=test_x['fullVisitorId'].copy()
test_x=test_x.drop(['fullVisitorId'],axis=1 ).astype(int)
 print('{} IV)CREATED TEST-DATASET FOR APPROACH 2 {}'.format('
df_train['keyword'].value_counts()<count_values>

 try:
del(test_df)
 del(tmp)
 del(val_df)
 except:
pass
submission_catboost_preds = np.zeros(len(test_x))
submission_xgb_preds = np.zeros(len(test_x))
submission_lgb_preds = np.zeros(len(test_x))
kfold=list(range(time_from.shape[0]-1))
kfold.remove(4)
 for fold in kfold:
fold_df = return_aggregated_df(full_df, fold, verbose=False)
 train_y = fold_df['totals.transactionRevenue']
train_x = fold_df.copy().drop(['fullVisitorId','totals.transactionRevenue'],inplace=False,axis=1 ).astype(int)
 print('
{} {}.1)CREATED TRAIN SET OF <{}> FOR THIS FOLD {}'.format(26*'
try:
del(fold_df)
 except:
pass
print('
{} {}.2)TRAINING CatBoostRegressor FOR THIS FOLD {}'.format(26*'
catboost = CatBoostRegressor(iterations=1100,learning_rate=0.04, depth=10, bagging_temperature = 0.2, od_wait=20,
eval_metric='RMSE', random_seed=0, od_type='Iter')
 catboost.fit(train_x , train_y, eval_set=[(validation_x, validation_y)],early_stopping_rounds=100, use_best_model=True, verbose=100)
 predictions = catboost.predict(validation_x)
 predictions[predictions<0]=0
print('{}{}.3)CATBOOST REGRESSION LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y, predictions)**.5, '<'*26))
submission_preds_raw=catboost.predict(test_x)
 submission_preds_raw[submission_preds_raw<0]=0
submission_catboost_preds += submission_preds_raw / len(kfold)
 print('
{} {}.4)TRAINING XGBRegressor FOR THIS FOLD {}'.format(26*'
xgb = XGBRegressor(n_estimators=1000, max_depth=20, learning_rate=0.07, booster='gbtree', random_state=0, importance_type='total_gain',
objective= 'reg:squarederror', gamma=1.45, alpha=0.0, subsample=0.67, colsample_bytree=0.054, colsample_bylevel= 0.50)
 xgb.fit(train_x , train_y ,eval_set=[(validation_x, validation_y)], early_stopping_rounds=100, eval_metric='rmse', verbose=100)
 predictions = xgb.predict(validation_x)
 predictions[predictions<0]=0
print('{}{}.5)XGB LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y, predictions)**.5, '<'*26))
submission_preds_raw=xgb.predict(test_x)
 submission_preds_raw[submission_preds_raw<0]=0
submission_xgb_preds += submission_preds_raw / len(kfold)
 print('
{} {}.6)TRAINING LGBRegressor FOR THIS FOLD {}'.format(26*'
lgb_reg = lgb.LGBMRegressor(n_estimators=1100, objective='regression', max_depth=-1, learning_rate=0.08, min_child_samples=40, boosting='gbdt',
feature_fraction=0.9, bagging_freq=10, bagging_fraction=0.8, bagging_seed=3, metric='rmse', lambda_l1=0, lambda_l2=1, verbosity=100)
 lgb_reg.fit(train_x, train_y, eval_set= [(validation_x, validation_y)], eval_metric='rmse', verbose=100 ,early_stopping_rounds = 100)
 predictions = lgb_reg.predict(validation_x.values, num_iteration=lgb_reg.best_iteration_)
 predictions[predictions<0]=0
print('{}{}.7)LGB LOSS = {}{}
'.format('>'*26, fold, mean_squared_error(validation_y.values, predictions)**.5, '<'*26))
submission_preds_raw=lgb_reg.predict(test_x.values, num_iteration=lgb_reg.best_iteration_)
 submission_preds_raw[submission_preds_raw<0]=0
submission_lgb_preds += submission_preds_raw / len(kfold )
df_train['location'].value_counts()<define_variables>
submissions_df['PredictedLogRevenue'] = submission_lgb_preds*0.4 + submission_catboost_preds*0.5 + submission_xgb_preds*0.1
submissions_df.to_csv('result.csv', index=False)
 print("Done" )