Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<count_duplicates>
imp_feature = ['uid3_count_dist', 'feat1', 'TransactionAmt', 'feat2', 'card1_count_dist', 'card2_count_dist', 'addr1_count_dist', 'uid_count_dist', 'uid2_count_dist', 'Transaction_hour', 'uid1_count_dist', 'P_emaildomain_count_dist', 'M_na', 'M_na_count_dist', 'D15_count_dist', 'Transaction_day_of_week', 'D1', 'card5_count_dist', 'dist1_count_dist', 'C13', 'D10_count_dist', 'id_20_count_dist', 'D4_count_dist', 'id_19_count_dist', 'C13_count_dist', 'D2_count_dist', 'id_31_count_dist', 'D1_count_dist', 'DeviceInfo_count_dist', 'R_emaildomain_count_dist', 'C1', 'uid4_count_dist', 'id_33_count_dist', 'V310_count_dist', 'C2', 'C1_count_dist', 'V307_count_dist', 'C14', 'C6', 'C2_count_dist', 'C14_count_dist', 'C6_count_dist', 'id_30_count_dist', 'D3_count_dist', 'V313_count_dist', 'D11_count_dist', 'D5_count_dist', 'id_05_count_dist', 'D9_count_dist', 'C11', 'dist2_count_dist', 'C9', 'V315_count_dist', 'id_13_count_dist', 'C9_count_dist', 'id_01_count_dist', 'V130_count_dist', 'M4_count_dist', 'C11_count_dist', 'V314_count_dist', 'M6_count_dist', 'M5_count_dist', 'card4_count_dist', 'V127_count_dist', 'V312_count_dist', 'D14_count_dist', 'V308_count_dist', 'C5', 'V283_count_dist', 'id_14_count_dist', 'V83_count_dist', 'M3_count_dist', 'C5_count_dist', 'V62_count_dist', 'id_18_count_dist', 'ProductCD_count_dist', 'V87_count_dist', 'V282_count_dist', 'V317_count_dist', 'D6_count_dist', 'card6_count_dist', 'V285_count_dist', 'V76_count_dist', 'card3_count_dist', 'id_02_count_dist', 'V45_count_dist', 'V143_count_dist', 'V38_count_dist', 'V131_count_dist', 'C10']
final_df = final_df[~final_df.index.duplicated(keep='first')]
final_df = final_df.rename(index=str, columns={"predicted": "PredictedLogRevenue"} )<count_values>
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
final_df.PredictedLogRevenue.value_counts(bins=3 )<data_type_conversions>
sub = pd.read_csv(".. /input/ieee-fraud-detection/sample_submission.csv" )
final_df = final_df.fillna(0); final_df.iloc[262153]<save_to_csv>
train_id = pd.read_csv(".. /input/ieee-fraud-detection/train_identity.csv")
 train_tr = pd.read_csv(".. /input/ieee-fraud-detection/train_transaction.csv" )
final_df.to_csv("sub.csv" )<set_options>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('fivethirtyeight')
 print(os.listdir(".. /input"))<load_from_csv>
test_id = pd.read_csv(".. /input/ieee-fraud-detection/test_identity.csv")
 test_tr = pd.read_csv(".. /input/ieee-fraud-detection/test_transaction.csv" )
def load_df(csv_path='.. /input/ga-customer-revenue-prediction/train.csv', nrows=None):
COLUMNS_HAVING_JSON = ['device', 'geoNetwork', 'totals', 'trafficSource']
df_json_columns = pd.read_csv(csv_path,
converters={column: json.loads for column in COLUMNS_HAVING_JSON},
dtype={'fullVisitorId': 'str'},
nrows=nrows)
 for column in COLUMNS_HAVING_JSON:
df_column = json_normalize(df_json_columns[column])
 df_column.columns = [f"{column}_{key}" for key in df_column.columns]
df_json_columns = df_json_columns.drop(column, axis=1)
 df_json_columns = df_json_columns.merge(df_column, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df_json_columns.shape}")
 return df_json_columns<load_from_csv>
train = pd.merge(train_tr, train_id, on='TransactionID', how='left')
 test = pd.merge(test_tr, test_id, on='TransactionID', how='left')
 del test_id, test_tr
del train_id, train_tr
gc.collect()
training_df = load_df(".. /input/ga-customer-revenue-prediction/train.csv")
 test_df = load_df(".. /input/ga-customer-revenue-prediction/test.csv" )<load_pretrained>
train_pos = train[train['isFraud']==1]
train_neg = train[train['isFraud']==0]
train_neg = train_neg.sample(2*int(train_pos.shape[0]), random_state=42)
 train = pd.concat([train_pos,train_neg] ).sort_index()
training_df.to_feather('train_data.feather')
 test_df.to_feather('test_data.feather')
 print("Loaded data in feather file" )<load_from_disk>
l = 3*int(train_pos.shape[0] )
df = pd.read_feather('train_data.feather')
 test_df = pd.read_feather('test_data.feather')
 print("Loaded from feather file" )<drop_column>
del train_pos
del train_neg
constant_valued_columns = [c for c in df.columns if df[c].nunique(dropna=False)==1]
non_relevant = ["visitNumber", "date", "fullVisitorId", "sessionId", "visitId", "visitStartTime"]
df = df.drop(constant_valued_columns, axis=1)
 test_df = test_df.drop(constant_valued_columns, axis=1)
 df=df.drop(['trafficSource_campaignCode'],axis=1 )<data_type_conversions>
train.isna().sum()
def convert_totals_columns_to_float(df,isTrain=True):
numeric_cols_to_transform = ['totals_hits'
,'totals_pageviews'
,'totals_newVisits'
,'totals_bounces']
if isTrain == True:
df[numeric_cols_to_transform] = df[numeric_cols_to_transform].fillna(0)
 df['totals_transactionRevenue'] = df['totals_transactionRevenue'].fillna(0)
 else:
df[numeric_cols_to_transform] = df[numeric_cols_to_transform].fillna(0)
 for col in numeric_cols_to_transform:
df[col] = df[col].astype('float32')
 if isTrain == True:
df['totals_transactionRevenue'] = df['totals_transactionRevenue'].astype('float32')
 return df<data_type_conversions>
train = train.sort_values('TransactionDT' )
df['totals_transactionRevenue'] = df['totals_transactionRevenue'].astype('float32')
 df=df.drop(['trafficSource_campaignCode'],axis=1 )<categorify>
train.drop_duplicates(inplace=True )
categorical_columns = [c for c in df.columns if not c.startswith("total")]
categorical_columns = [c for c in categorical_columns if c not in constant_valued_columns + non_relevant]
for c in categorical_columns:
le = LabelEncoder()
train_vals = list(df[c].values.astype(str))
test_vals = list(test_df[c].values.astype(str))
le.fit(train_vals + test_vals)
 df[c] = le.transform(train_vals)
 test_df[c] = le.transform(test_vals )<data_type_conversions>
target = train["isFraud"]
train.drop(["isFraud"], axis=1, inplace=True )
df = convert_totals_columns_to_float(df)
 test_df = convert_totals_columns_to_float(test_df,False )<data_type_conversions>
useful_features = [col for col in train.columns]
def normalize_numerical_columns(df, isTrain = True):
df["totals_hits"] = df["totals_hits"].astype(float)
 df["totals_hits"] =(df["totals_hits"] - min(df["totals_hits"])) /(max(df["totals_hits"])- min(df["totals_hits"]))
df["totals_pageviews"] = df["totals_pageviews"].astype(float)
 df["totals_pageviews"] =(df["totals_pageviews"] - min(df["totals_pageviews"])) /(max(df["totals_pageviews"])- min(df["totals_pageviews"]))
if isTrain:
df["totals_transactionRevenue"] = df["totals_transactionRevenue"].fillna(0.0)
 return df<drop_column>
train.isna().sum()
df = normalize_numerical_columns(df)
 test_df = normalize_numerical_columns(test_df, isTrain = False )<data_type_conversions>
pd.set_option('display.max_columns', None )
features = [c for c in df.columns if c not in constant_valued_columns + non_relevant]
features.remove("totals_transactionRevenue")
 df_new["totals_transactionRevenue"] = np.log1p(df["totals_transactionRevenue"].astype(float))<sort_values>
train = train.iloc[:, 1:31]
test = test.iloc[:, 1:31]
df_count_nonzero_tr = df.groupby('fullVisitorId')['totals_transactionRevenue'].agg(['count'])
 df_count_nonzero_tr.columns = ["Count"]
totalNonZeroTransactionCount = df_count_nonzero_tr["Count"].sum()
df_count_nonzero_tr['buying_probability'] = df_count_nonzero_tr["Count"]/totalNonZeroTransactionCount
df_count_nonzero_tr=df_count_nonzero_tr.sort_values(by=['buying_probability'],ascending=False)
 print(df_count_nonzero_tr.head(10))
probSum = df_count_nonzero_tr['buying_probability'].sum()
print(probSum )<split>
train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
train_x, valid_x, train_y, valid_y = train_test_split(df[features], df_new["totals_transactionRevenue"], test_size=0.20, random_state=42 )<create_dataframe>
train['uid'] = train['card1'].astype(str)+'_'+train['card2'].astype(str)
 test['uid'] = test['card1'].astype(str)+'_'+test['card2'].astype(str)
 train['uid1'] = train['uid'].astype(str)+'_'+train['card3'].astype(str)
 test['uid1'] = test['uid'].astype(str)+'_'+test['card3'].astype(str)
 train['uid2'] = train['uid'].astype(str)+'_'+train['card3'].astype(str)+'_'+train['card5'].astype(str)
 test['uid2'] = test['uid'].astype(str)+'_'+test['card3'].astype(str)+'_'+test['card5'].astype(str)
 train['uid3'] = train['uid2'].astype(str)+'_'+train['addr1'].astype(str)+'_'+train['addr2'].astype(str)
 test['uid3'] = test['uid2'].astype(str)+'_'+test['addr1'].astype(str)+'_'+test['addr2'].astype(str)
 train['uid4'] = train['card4'].astype(str)+'_'+train['card6'].astype(str)
 test['uid4'] = test['card4'].astype(str)+'_'+test['card6'].astype(str)
 train['TransactionAmt_check'] = np.where(train['TransactionAmt'].isin(test['TransactionAmt']), 1, 0)
 test['TransactionAmt_check'] = np.where(test['TransactionAmt'].isin(train['TransactionAmt']), 1, 0)
lgb_params = {"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 200,
"min_child_samples" : 100,
"learning_rate" : 0.05,
"boosting_type" : "gbdt",
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1}
lgb_train = lgb.Dataset(train_x, label=train_y)
 lgb_val = lgb.Dataset(valid_x, label=valid_y)
 model = lgb.train(lgb_params, lgb_train, 700, valid_sets=[lgb_val], early_stopping_rounds=150 )<define_variables>
train = pd.concat([train, test] )
unimportant_features = ["visitNumber", "date", "fullVisitorId", "sessionId", "visitId", "visitStartTime"]
features = [c for c in df.columns if c not in constant_valued_columns + unimportant_features]
print(features )<save_to_csv>
train['M_na'] = abs(train.isna().sum(axis=1 ).astype(np.int8))
features = [c for c in df.columns if c not in constant_valued_columns + unimportant_features]
features.remove("totals_transactionRevenue")
 preds = model.predict(test_df[features], num_iteration=model.best_iteration)
 test_df["PredictedLogRevenue"] = np.expm1(preds)
 result = test_df.groupby("fullVisitorId" ).agg({"PredictedLogRevenue" : "sum"} ).reset_index()
result["PredictedLogRevenue"] = np.log1p(result["PredictedLogRevenue"])
 result["PredictedLogRevenue"] = result["PredictedLogRevenue"].apply(lambda x : 0.0 if x < 0 else x)
 result["PredictedLogRevenue"] = result["PredictedLogRevenue"].fillna(0)
 result.to_csv("baseline.csv", index=False)
 print(result.isnull().values.any())
 result.head()<split>
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = 'Others'
gc.collect()
return dataframe
def calculate_rmse(df_new):
temp_df = pd.DataFrame()
temp_df["totals_transactionRevenue"] = np.log1p(df_new["totals_transactionRevenue"].astype(float))
train_x, valid_x, train_y, valid_y = train_test_split(df_new[features], temp_df["totals_transactionRevenue"], test_size=0.2, random_state=20)
 lgb_params = {"objective" : "regression", "metric" : "rmse",
"num_leaves" : 50, "learning_rate" : 0.02,
"bagging_fraction" : 0.75, "feature_fraction" : 0.8, "bagging_frequency" : 9}
lgb_train = lgb.Dataset(train_x, label=train_y)
 lgb_val = lgb.Dataset(valid_x, label=valid_y)
 model = lgb.train(lgb_params, lgb_train, 400, valid_sets=[lgb_val], early_stopping_rounds=100, verbose_eval=200)
 prediction = model.predict(valid_x, num_iteration=model.best_iteration)
 rmse = np.sqrt(metrics.mean_squared_error(prediction, valid_y))
return rmse<randomize_order>
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int )
testing_cols = ['trafficSource_campaign','totals.pageviews','trafficSource.adContent']
for cols in testing_cols:
df_new = df.copy()
count = 0
i=1
while i < 500:
df_new[cols] = np.random.permutation(df_new[cols])
 if calculate_rmse(df_new)<bestValue:
count=count+1
print(count)
 print("For ",cols," number of times RMSE is lesser than base is : ",count )<import_modules>
neglect = ["TransactionAmt", 'Transaction_day_of_week', 'Transaction_hour', "TransactionAmt_Log", 'TransactionAmt_decimal']
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestRegressor as RFF
import xgboost as xgb
from pprint import pprint
import math
from scipy.stats import kurtosis, skew 
<set_options>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other',
'optonline.net': 'other', 'hotmail.co.uk': 'microsoft', 'comcast.net': 'other',
'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo', 'yahoo.es': 'yahoo',
'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol',
'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink', 'gmail.com': 'google',
'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other', 'web.de': 'other',
'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo', 'servicios-ta.com': 'other',
'netzero.net': 'other', 'suddenlink.net': 'other', 'roadrunner.com': 'other',
'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att',
'frontier.com': 'yahoo', 'anonymous.com': 'other', 'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo', 'ymail.com': 'yahoo',
'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft',
'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
plt.rcParams['figure.figsize'] =(12,6 )<categorify>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df 
<load_from_csv>
useful_features = [col for col in train.columns if col not in neglect]
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )<count_missing_values>
print(f'<compute_test_metric>
dropping =["D8_count_dist", "V138_count_dist", "V139_count_dist", "V140_count_dist", "V141_count_dist",\
"V146_count_dist", "V147_count_dist", "V148_count_dist", "V149_count_dist", "V144_count_dist",\
"V145_count_dist", "V150_count_dist", "V151_count_dist", "V152_count_dist", "V153_count_dist",\
"V154_count_dist", "V155_count_dist", "V156_count_dist", "V157_count_dist", "V158_count_dist",\
"V159_count_dist", "V160_count_dist", "V161_count_dist", "V162_count_dist", "V163_count_dist",\
"V164_count_dist", "V165_count_dist", "V166_count_dist", "V168_count_dist", "V170_count_dist",\
"V171_count_dist", "V172_count_dist", "V173_count_dist", "V174_count_dist", "V175_count_dist",\
"V176_count_dist", "V177_count_dist", "V178_count_dist", "V179_count_dist", "V180_count_dist",\
"V181_count_dist", "V182_count_dist", "V183_count_dist", "V184_count_dist", "V185_count_dist",\
"V186_count_dist", "V187_count_dist", "V188_count_dist", "V189_count_dist", "V190_count_dist",\
"V191_count_dist", "V192_count_dist", "V193_count_dist", "V194_count_dist", "V195_count_dist",\
"V196_count_dist", "V197_count_dist", "V198_count_dist", "V199_count_dist", "V200_count_dist",\
"V201_count_dist", "V202_count_dist", "V203_count_dist", "V204_count_dist", "V205_count_dist",\
"V206_count_dist", "V207_count_dist", "V208_count_dist", "V209_count_dist", "V210_count_dist",\
"V211_count_dist", "V212_count_dist", "V213_count_dist", "V214_count_dist", "V215_count_dist",\
"V216_count_dist", "V218_count_dist", "V219_count_dist", "V221_count_dist", "V222_count_dist",\
"V223_count_dist", "V224_count_dist", "V225_count_dist", "V226_count_dist", "V227_count_dist",\
"V228_count_dist", "V229_count_dist", "V230_count_dist", "V231_count_dist", "V232_count_dist",\
"V233_count_dist", "V234_count_dist", "V235_count_dist", "V236_count_dist", "V237_count_dist",\
"V205_count_dist", "V205_count_dist", "V205_count_dist", "V205_count_dist", "V205_count_dist",\
"V238_count_dist", "V239_count_dist", "V240_count_dist", "V241_count_dist", "V242_count_dist",\
"V243_count_dist", "V244_count_dist", "V245_count_dist","V246_count_dist", "V247_count_dist",\
"V248_count_dist", "V249_count_dist", "V250_count_dist", "V251_count_dist", "V252_count_dist",\
"V253_count_dist", "V254_count_dist", "V255_count_dist", "V256_count_dist", "V257_count_dist",\
"V258_count_dist", "V259_count_dist", "V260_count_dist", "V261_count_dist", "V262_count_dist",\
"V263_count_dist", "V264_count_dist", "V265_count_dist", "V266_count_dist", "V267_count_dist",\
"V268_count_dist", "V269_count_dist", "V270_count_dist", "V271_count_dist", "V272_count_dist",\
"V273_count_dist", "V274_count_dist", "V275_count_dist", "V276_count_dist", "V277_count_dist",\
"V278_count_dist", "V323_count_dist", "V324_count_dist", "V325_count_dist", "V326_count_dist",\
"V327_count_dist", "V328_count_dist", "V329_count_dist", "V330_count_dist", "V331_count_dist",\
"V332_count_dist", "V333_count_dist", "V334_count_dist", "V335_count_dist", "V336_count_dist",\
"V237_count_dist", "V238_count_dist", "V239_count_dist", "id_04_count_dist", "id_06_count_dist",\
"id_08_count_dist", "id_10_count_dist", "id_22_count_dist", "id_27_count_dist", "id_29_count_dist",\
"id_36_count_dist", "id_37_count_dist", "id_38_count_dist"]
def rmse(y_true, y_pred):
return round(np.sqrt(mean_squared_error(y_true, y_pred)) , 5)
 def split_data(df=df_train,rate=.8):
df = df.sort_values('date' ).copy()
df.drop(['fullVisitorId','visitId','visitStartTime'],axis=1,inplace=True)
 df['Revenue'] = np.log1p(df['Revenue'])
 global X_train,X_valid,y_train,y_valid
n_train = int(len(df)*rate)
 X_train = df.drop(['Revenue','date'],axis=1 ).iloc[:n_train]
X_valid = df.drop(['Revenue','date'],axis=1 ).iloc[n_train:]
y_train = df['Revenue'].iloc[:n_train]
y_valid = df['Revenue'].iloc[n_train:]
print(X_train.shape,X_valid.shape)
 def encode_data(verbose=False):
global df_train_encoded,df_test_encoded
df_train_encoded = df_train.copy()
df_test_encoded = df_test.copy()
for col in df_train.columns:
if df_train_encoded[col].dtype == 'object' and col not in ['fullVisitorId','visitId','visitStartTime','date']:
if verbose:
print(col)
 lb = LabelEncoder()
lb.fit(list(df_train_encoded[col].unique())+ list(df_test_encoded[col].unique()))
df_train_encoded[col] = lb.transform(df_train_encoded[col])
 df_test_encoded[col] = lb.transform(df_test_encoded[col])
 def run_xgb() :
params = {
'objective':'reg:linear',
'eval_metric':'rmse',
'learning_rate':.01,
'eta': 0.15,
'subsample': 0.6,
'colsample_bytree': 0.6,
'lambda':1,
'random_state': 42,
'silent':True
}
params['n_thread'] = -1
params['max_depth'] = 8
params['min_child_weight'] = 100
params['gamma'] = 5
params['subsample'] = 1
params['colsample_bytree'] =.95
params['colsample_bylevel'] = 0.35
params['alpha'] = 25
params['lambda'] = 25
xgb_train_data = xgb.DMatrix(X_train, y_train)
 xgb_val_data = xgb.DMatrix(X_valid, y_valid)
 model = xgb.train(params, xgb_train_data,
num_boost_round = 200,
evals= [(xgb_train_data, 'train'),(xgb_val_data, 'valid')],
early_stopping_rounds=50,
verbose_eval=20 
)
 return model
def submit() :
test_matrix = xgb.DMatrix(X_test)
 y_pred = clf.predict(test_matrix,ntree_limit=clf.best_ntree_limit)
 df_test['PredictedLogRevenue'] = y_pred
engineer_prediction
print('rmse after engineer prediction')
 print(rmse(y_pred,df_test['PredictedLogRevenue']))
submit = df_test[['PredictedLogRevenue','fullVisitorId']].groupby('fullVisitorId' ).PredictedLogRevenue.sum().reset_index()
submit.to_csv('submit.csv',index=False)
 test(y_pred)
 def engineer_prediction(df_test):
df_test[df_test['totals_hits'] == 1].PredictedLogRevenue = 0
df_test[df_test['totals_timeOnSite'] == 0].PredictedLogRevenue = 0
df_test[df_test['totals_bouces'] == 1].PredictedLogRevenue = 0
return dftest
def test(predict):
y_test = np.log1p(df_test['totals_transactionRevenue'])
 print(rmse(y_test,predict)) 
<feature_engineering>
def prepare_data(df_train,df_test,
del_col=['fullVisitorId','visitId','visitStartTime','date'],to_log=None):
df_train = df_train.sort_values('date' ).copy()
df_train = df_train.drop(del_col,axis=1 ).copy()
df_test = df_test.drop(del_col,axis=1 ).copy()
if to_log is not None:
df_train[to_log] = np.log1p(df_train[to_log])
 df_test[to_log] = np.log1p(df_test[to_log])
 df_train['totals_transactionRevenue'] = np.log1p(df_train['totals_transactionRevenue'])
 df_test['totals_transactionRevenue'] = np.log1p(df_test['totals_transactionRevenue'])
 global X_train,X_valid,y_train,y_valid,X_test,y_test
n_train = int(len(df_train)*.8)
 X_train = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[:n_train]
X_valid = df_train.drop(['totals_transactionRevenue'],axis=1 ).iloc[n_train:]
y_train = df_train['totals_transactionRevenue'].iloc[:n_train]
y_valid = df_train['totals_transactionRevenue'].iloc[n_train:]
X_test = df_test.drop(['totals_transactionRevenue'],axis=1)
 y_test = df_test['totals_transactionRevenue'] 
<feature_engineering>
def feature_engineering(df):
df = df.copy()
df['month'] = df['date'].dt.month
df['day'] = df['date'].dt.day
df['weekday'] = df['date'].dt.weekday
df['weekofyear'] = df['date'].dt.weekofyear
df['browser_category'] = df['device_browser'] + '_' + df['device_deviceCategory']
df['browser_operatingSystem'] = df['device_browser'] + '_' + df['device_operatingSystem']
df['month_unique_user_count'] = df.groupby('month')['fullVisitorId'].transform('nunique')
 df['day_unique_user_count'] = df.groupby('day')['fullVisitorId'].transform('nunique')
 df['weekday_unique_user_count'] = df.groupby('weekday')['fullVisitorId'].transform('nunique')
 df['weekofyear_unique_user_count'] = df.groupby('weekofyear')['fullVisitorId'].transform('nunique')
 df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')
 df['sum_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')
 df['count_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')
 df['mean_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')
 df['mean_hits_per_day'] = df.groupby(['day'])['totals_hits'].transform('mean')
 df['sum_hits_per_day'] = df.groupby(['day'])['totals_hits'].transform('sum')
 df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')
 df['sum_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('sum')
 df['count_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('count')
 df['mean_pageviews_per_region'] = df.groupby('geoNetwork_region')['totals_pageviews'].transform('mean')
 df['sum_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')
 df['count_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')
 df['mean_hits_per_network_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')
 df['sum_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('sum')
 df['count_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('count')
 df['mean_hits_per_region'] = df.groupby('geoNetwork_region')['totals_hits'].transform('mean')
 df['sum_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('sum')
 df['count_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('count')
 df['mean_hits_per_country'] = df.groupby('geoNetwork_country')['totals_hits'].transform('mean')
 df['user_pageviews_sum'] = df.groupby('fullVisitorId')['totals_pageviews'].transform('sum')
 df['user_hits_sum'] = df.groupby('fullVisitorId')['totals_hits'].transform('sum')
 df['user_pageviews_count'] = df.groupby('fullVisitorId')['totals_pageviews'].transform('count')
 df['user_hits_count'] = df.groupby('fullVisitorId')['totals_hits'].transform('count')
 df['user_pageviews_sum_to_mean'] = df['user_pageviews_sum'] / df['user_pageviews_sum'].mean()
df['user_hits_sum_to_mean'] = df['user_hits_sum'] / df['user_hits_sum'].mean()
df['user_pageviews_to_region'] = df['user_pageviews_sum'] / df['mean_pageviews_per_region']
df['user_hits_to_region'] = df['user_hits_sum'] / df['mean_hits_per_region']
return df<feature_engineering>
df_train = feature_engineering(df_train)
 df_test = feature_engineering(df_test )<categorify>
train.isna().sum()
encode_data(verbose=True)
 prepare_data(df_train_encoded,df_test_encoded,del_col=['fullVisitorId','visitId',
'visitStartTime','date'],to_log=['visitNumber','totals_hits','totals_pageviews',
'totals_sessionQualityDim','totals_timeOnSite','trafficSource_keyword'])
<compute_test_metric>
train.fillna(-333, inplace=True )
<import_modules>
X = train.iloc[:l, :]
test = train.iloc[l:, :]
import pandas as pd
import numpy as np
import ast<load_from_csv>
for f in X.columns:
if X[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X[f].values)+ list(test[f].values))
X[f] = lbl.transform(list(X[f].values))
test[f] = lbl.transform(list(test[f].values))
test = pd.read_csv('.. /input/test_v2.csv',dtype={'fullVisitorId': 'str'} )<load_from_csv>
y=target
submission_sample = pd.read_csv('.. /input/sample_submission_v2.csv',dtype={'fullVisitorId': 'str'})
 submission_sample.shape<prepare_output>
categorical_var = np.where(train.dtypes != np.float)[0]
print('
Categorical Variables indices : ',categorical_var )
test_prep = test.copy()
test_prep['PredictedLogRevenue'] = 0
submission_no_return = test_prep.groupby('fullVisitorId',as_index=False ).PredictedLogRevenue.sum()
submission_no_return.head()<save_to_csv>
del train
submission_no_return.to_csv('submission_no_return.csv',index=False )<prepare_output>
test_prep_2 = test.copy()
test_prep_2['Revenue'] = test_prep_2['totals'].apply(lambda x: ast.literal_eval(x ).get('transactionRevenue',np.nan))<data_type_conversions>
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
"n_jobs" : -1
}
test_prep_2['Revenue'] =pd.to_numeric(test_prep_2['Revenue'],errors='coerce' )<feature_engineering>
folds = TimeSeriesSplit(n_splits=5)
 aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
training_start_time = time()
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, y)) :
start_time = time()
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 print('Fold {} finished in {}'.format(fold + 1, str(datetime.timedelta(seconds=time() - start_time))))
print('-' * 30)
 print('Training has finished.')
 print('Total training time is {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))
print('Mean AUC:', np.mean(aucs))
print('-' * 30 )
submission_right_answer = test_prep_2.groupby('fullVisitorId',as_index=False ).Revenue.sum()
submission_right_answer['PredictedLogRevenue'] = submission_right_answer.Revenue.apply(lambda x: np.log(x+1))
submission_right_answer.drop(columns='Revenue',inplace=True)
 submission_right_answer.head()<save_to_csv>
feature_importances['average'] = feature_importances[['fold_{}'.format(fold + 1)for fold in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;
submission_right_answer.to_csv('submission_right_answer.csv',index=False )<set_options>
best_iter = clf.best_iteration
%matplotlib inline
pd.options.display.max_columns = 999
plt.rcParams['figure.figsize'] =(14, 7)
 font = {'family' : 'verdana',
'weight' : 'bold',
'size' : 14}
plt.rc('font', **font)
 warnings.simplefilter("ignore")
 gc.enable()<load_from_csv>
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, y )
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 train.shape, test.shape<load_from_csv>
sub['isFraud'] = clf.predict_proba(test)[:, 1]
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'} )<data_type_conversions>
sub.to_csv('ieee_cis_fraud_detection_new.csv', index=False )