Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<predict_on_test>
init_notebook_mode(connected=True)
 print(os.listdir(".. /input"))
preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
df_id = pd.read_csv(".. /input/train_identity.csv")
 df_trans = pd.read_csv(".. /input/train_transaction.csv" )
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
for name in summary['Name'].value_counts().index:
summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)
 return summary
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def CalcOutliers(df_num):
data_mean, data_std = np.mean(df_num), np.std(df_num)
 cut = data_std * 3
lower, upper = data_mean - cut, data_mean + cut
outliers_lower = [x for x in df_num if x < lower]
outliers_higher = [x for x in df_num if x > upper]
outliers_total = [x for x in df_num if x < lower or x > upper]
outliers_removed = [x for x in df_num if x > lower and x < upper]
print('Identified lowest outliers: %d' % len(outliers_lower))
print('Identified upper outliers: %d' % len(outliers_higher))
print('Total outlier observations: %d' % len(outliers_total))
print('Non-outlier observations: %d' % len(outliers_removed))
print("Total percentual of Outliers: ", round(( len(outliers_total)/ len(outliers_removed)) *100, 4))
return
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
df_trans = reduce_mem_usage(df_trans)
 df_id = reduce_mem_usage(df_id )
preds_test = predict_result(model,x_test,img_size_target )<save_to_csv>
df_trans['TransactionAmt'] = df_trans['TransactionAmt'].astype(float)
 print("Transaction Amounts Quantiles:")
 print(df_trans['TransactionAmt'].quantile([.01,.025,.1,.25,.5,.75,.9,.975,.99]))
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
print("Card Features Quantiles: ")
 print(df_trans[['card1', 'card2', 'card3', 'card5']].quantile([0.01,.025,.1,.25,.5,.75,.975,.99]))
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<set_options>
df_trans.loc[df_trans.card3.isin(df_trans.card3.value_counts() [df_trans.card3.value_counts() < 200].index), 'card3'] = "Others"
df_trans.loc[df_trans.card5.isin(df_trans.card5.value_counts() [df_trans.card5.value_counts() < 300].index), 'card5'] = "Others"
%matplotlib inline
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.filterwarnings("ignore" )<load_from_csv>
print("Card Features Quantiles: ")
 print(df_trans[['addr1', 'addr2']].quantile([0.01,.025,.1,.25,.5,.75,.90,.975,.99]))
data = pd.read_csv(".. /input/tmdb-box-office-prediction/train.csv")
 print(data.shape)
 data.head(n=2 )<create_dataframe>
df_trans.loc[df_trans.addr1.isin(df_trans.addr1.value_counts() [df_trans.addr1.value_counts() <= 5000 ].index), 'addr1'] = "Others"
df_trans.loc[df_trans.addr2.isin(df_trans.addr2.value_counts() [df_trans.addr2.value_counts() <= 50 ].index), 'addr2'] = "Others"
data_explore = data.copy()<count_missing_values>
df_trans.loc[df_trans['P_emaildomain'].isin(['gmail.com', 'gmail']),'P_emaildomain'] = 'Google'
df_trans.loc[df_trans['P_emaildomain'].isin(['yahoo.com', 'yahoo.com.mx', 'yahoo.co.uk',
'yahoo.co.jp', 'yahoo.de', 'yahoo.fr',
'yahoo.es']), 'P_emaildomain'] = 'Yahoo Mail'
df_trans.loc[df_trans['P_emaildomain'].isin(['hotmail.com','outlook.com','msn.com', 'live.com.mx',
'hotmail.es','hotmail.co.uk', 'hotmail.de',
'outlook.es', 'live.com', 'live.fr',
'hotmail.fr']), 'P_emaildomain'] = 'Microsoft'
df_trans.loc[df_trans.P_emaildomain.isin(df_trans.P_emaildomain\ 
.value_counts() [df_trans.P_emaildomain.value_counts() <= 500 ]\ 
.index), 'P_emaildomain'] = "Others"
df_trans.P_emaildomain.fillna("NoInf", inplace=True )
data_explore.isna().sum()<data_type_conversions>
ploting_cnt_amt(df_trans, 'P_emaildomain' )
data_explore['is_sequel'] = data_explore['belongs_to_collection'].apply(lambda x: 0 if pd.isna(x)else 1 ).astype('int64' )<feature_engineering>
df_trans.loc[df_trans['R_emaildomain'].isin(['gmail.com', 'gmail']),'R_emaildomain'] = 'Google'
df_trans.loc[df_trans['R_emaildomain'].isin(['yahoo.com', 'yahoo.com.mx', 'yahoo.co.uk',
'yahoo.co.jp', 'yahoo.de', 'yahoo.fr',
'yahoo.es']), 'R_emaildomain'] = 'Yahoo Mail'
df_trans.loc[df_trans['R_emaildomain'].isin(['hotmail.com','outlook.com','msn.com', 'live.com.mx',
'hotmail.es','hotmail.co.uk', 'hotmail.de',
'outlook.es', 'live.com', 'live.fr',
'hotmail.fr']), 'R_emaildomain'] = 'Microsoft'
df_trans.loc[df_trans.R_emaildomain.isin(df_trans.R_emaildomain\ 
.value_counts() [df_trans.R_emaildomain.value_counts() <= 300 ]\ 
.index), 'R_emaildomain'] = "Others"
df_trans.R_emaildomain.fillna("NoInf", inplace=True )
def modify_date(x):

 x=str(x)
 year=x.split('/')[2]
if int(year)<20:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
data_explore['release_date']=data_explore['release_date'].apply(lambda x: modify_date(x))
data_explore['release_year'] = pd.DatetimeIndex(data_explore['release_date'] ).year
data_explore['release_month'] = pd.DatetimeIndex(data_explore['release_date'] ).month
data_explore['release_day'] = pd.DatetimeIndex(data_explore['release_date'] ).day
data_explore['release_dow'] = pd.DatetimeIndex(data_explore['release_date'] ).dayofweek<drop_column>
ploting_cnt_amt(df_trans, 'R_emaildomain' )
drop_cols = ['id', 'belongs_to_collection', 'homepage', 'imdb_id', 'release_date', 'poster_path', 'tagline', 'title']
data_explore = data_explore.drop(columns=drop_cols, axis=1 )<count_missing_values>
df_trans.loc[df_trans.C1.isin(df_trans.C1\ 
.value_counts() [df_trans.C1.value_counts() <= 400 ]\ 
.index), 'C1'] = "Others"
nan_cols = data_explore.isna().sum()
nan_cols[nan_cols>0]<feature_engineering>
df_trans.loc[df_trans.C2.isin(df_trans.C2\ 
.value_counts() [df_trans.C2.value_counts() <= 350 ]\ 
.index), 'C2'] = "Others"
dict_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
for col in dict_cols:
data_explore[col] = data_explore[col].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))<data_type_conversions>
ploting_cnt_amt(df_trans, 'C2' )
na_cols = data_explore.columns[data_explore.isna().any() ].tolist()
na_cols.remove('overview')
 na_cols.remove('runtime')
 data_explore['runtime'].fillna(value=data_explore['runtime'].median() , inplace=True)
 data_explore['overview'].fillna(value='', inplace=True)
 for col in na_cols:
data_explore[col].fillna(value='', inplace=True )<categorify>
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 df_trans["Date"] = df_trans['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 df_trans['_Weekdays'] = df_trans['Date'].dt.dayofweek
df_trans['_Hours'] = df_trans['Date'].dt.hour
df_trans['_Days'] = df_trans['Date'].dt.day
def get_names(x, col):

 names = []
for item in x:
if col=='crew':
if item['job']=='Director':
names.append(item['name'])
 elif col=='cast':
if item['order'] in(0, 1, 2):
names.append(item['name'])
 else:
names.append(item['name'])
 return names
for col in dict_cols:
data_explore[col] = data_explore[col].apply(lambda x: get_names(x, col))<define_variables>
color_op = ['
'
'
dates_temp = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].count().reset_index()
trace = go.Scatter(x=dates_temp['Date'], y=dates_temp.TransactionAmt,
opacity = 0.8, line = dict(color = color_op[7]), name= 'Total Transactions')
 dates_temp_sum = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].sum().reset_index()
trace1 = go.Scatter(x=dates_temp_sum.Date, line = dict(color = color_op[1]), name="Total Amount",
y=dates_temp_sum['TransactionAmt'], opacity = 0.8, yaxis='y2')
 layout = dict(
 title= "Total Transactions and Fraud Informations by Date",
xaxis=dict(
 rangeselector=dict(
 buttons=list([
dict(count=1, label='1m', step='month', stepmode='backward'),
dict(count=3, label='3m', step='month', stepmode='backward'),
dict(count=6, label='6m', step='month', stepmode='backward'),
dict(step='all')
 ])
),
rangeslider=dict(visible = True),
type='date'),
yaxis=dict(title='Total Transactions'),
yaxis2=dict(overlaying='y',
anchor='x', side='right',
zeroline=False, showgrid=False,
title='Total Transaction Amount')
)
 fig = dict(data= [trace, trace1,], layout=layout)
 iplot(fig )
Q1 = data_explore.quantile(0.25)
 Q3 = data_explore.quantile(0.75)
 IQR = Q3 - Q1
outliers =(( data_explore <(Q1 - 1.5 * IQR)) |(data_explore >(Q3 + 1.5 * IQR)) ).sum()
outliers[outliers>0]<feature_engineering>
color_op = ['
'
'
tmp_amt = df_trans.groupby([df_trans.Date.dt.date, 'isFraud'])['TransactionAmt'].sum().reset_index()
tmp_trans = df_trans.groupby([df_trans.Date.dt.date, 'isFraud'])['TransactionAmt'].count().reset_index()
tmp_trans_fraud = tmp_trans[tmp_trans['isFraud'] == 1]
tmp_amt_fraud = tmp_amt[tmp_amt['isFraud'] == 1]
dates_temp = df_trans.groupby(df_trans.Date.dt.date)['TransactionAmt'].count().reset_index()
trace = go.Scatter(x=tmp_trans_fraud['Date'], y=tmp_trans_fraud.TransactionAmt,
opacity = 0.8, line = dict(color = color_op[1]), name= 'Fraud Transactions')
 trace1 = go.Scatter(x=tmp_amt_fraud.Date, line = dict(color = color_op[7]), name="Fraud Amount",
y=tmp_amt_fraud['TransactionAmt'], opacity = 0.8, yaxis='y2')
 layout = dict(
 title= "FRAUD TRANSACTIONS - Total Transactions and Fraud Informations by Date",
xaxis=dict(
 rangeselector=dict(
 buttons=list([
dict(count=1, label='1m', step='month', stepmode='backward'),
dict(count=3, label='3m', step='month', stepmode='backward'),
dict(count=6, label='6m', step='month', stepmode='backward'),
dict(step='all')
 ])
),
rangeslider=dict(visible = True),
type='date'),
yaxis=dict(title='Total Transactions'),
yaxis2=dict(overlaying='y',
anchor='x', side='right',
zeroline=False, showgrid=False,
title='Total Transaction Amount')
)
 fig = dict(data= [trace, trace1], layout=layout)
 iplot(fig )
most_popular_movies = data_explore.sort_values('popularity', ascending=False ).head(n=20)
 most_popular_movies['revenue(million)'] = most_popular_movies['revenue'].apply(lambda x : x//1000000)
 most_popular_movies['budget(million)'] = most_popular_movies['budget'].apply(lambda x : x//1000000)
 most_popular_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]<feature_engineering>
df_train = df_trans.merge(df_id, how='left', left_index=True, right_index=True )
highest_revenue_movies = data_explore.sort_values('revenue', ascending=False ).head(n=20)
 highest_revenue_movies['revenue(million)'] = highest_revenue_movies['revenue'].apply(lambda x : x//1000000)
 highest_revenue_movies['budget(million)'] = highest_revenue_movies['budget'].apply(lambda x : x//1000000)
 highest_revenue_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]<feature_engineering>
df_train.loc[df_train['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
df_train.loc[df_train['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
df_train.loc[df_train['id_30'].str.contains('Mac OS', na=False), 'id_30'] = 'Mac'
df_train.loc[df_train['id_30'].str.contains('Android', na=False), 'id_30'] = 'Android'
df_train['id_30'].fillna("NAN", inplace=True )
highest_budget_movies = data_explore.sort_values('budget', ascending=False ).head(n=20)
 highest_budget_movies['revenue(million)'] = highest_budget_movies['revenue'].apply(lambda x : x//1000000)
 highest_budget_movies['budget(million)'] = highest_budget_movies['budget'].apply(lambda x : x//1000000)
 highest_budget_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]<feature_engineering>
df_train.loc[df_train['id_31'].str.contains('chrome', na=False), 'id_31'] = 'Chrome'
df_train.loc[df_train['id_31'].str.contains('firefox', na=False), 'id_31'] = 'Firefox'
df_train.loc[df_train['id_31'].str.contains('safari', na=False), 'id_31'] = 'Safari'
df_train.loc[df_train['id_31'].str.contains('edge', na=False), 'id_31'] = 'Edge'
df_train.loc[df_train['id_31'].str.contains('ie', na=False), 'id_31'] = 'IE'
df_train.loc[df_train['id_31'].str.contains('samsung', na=False), 'id_31'] = 'Samsung'
df_train.loc[df_train['id_31'].str.contains('opera', na=False), 'id_31'] = 'Opera'
df_train['id_31'].fillna("NAN", inplace=True)
 df_train.loc[df_train.id_31.isin(df_train.id_31.value_counts() [df_train.id_31.value_counts() < 200].index), 'id_31'] = "Others"
most_profit_movies = data_explore.copy()
most_profit_movies['revenue(million)'] = most_profit_movies['revenue'].apply(lambda x : x//1000000)
 most_profit_movies['budget(million)'] = most_profit_movies['budget'].apply(lambda x : x//1000000)
 most_profit_movies['profit(million)'] = most_profit_movies['revenue(million)']-most_profit_movies['budget(million)']
most_profit_movies = most_profit_movies.sort_values('profit(million)', ascending=False ).head(n=20)
 most_profit_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)', 'profit(million)']]<define_variables>
df_trans = pd.read_csv('.. /input/train_transaction.csv')
 df_test_trans = pd.read_csv('.. /input/test_transaction.csv')
 df_id = pd.read_csv('.. /input/train_identity.csv')
 df_test_id = pd.read_csv('.. /input/test_identity.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 df_train = df_trans.merge(df_id, how='left', left_index=True, right_index=True, on='TransactionID')
 df_test = df_test_trans.merge(df_test_id, how='left', left_index=True, right_index=True, on='TransactionID')
 print(df_train.shape)
 print(df_test.shape)
 del df_trans, df_id, df_test_trans, df_test_id
genres_info = []
for col in genres:
total_movies, total_budget, median_budget, total_revenue, median_revenue, median_popularity=0, 0, 0, 0, 0, 0
total_movies = data_explore_genres[data_explore_genres[col]==1][col].count()
total_budget = data_explore_genres[data_explore_genres[col]==1]['budget'].sum()
median_budget = data_explore_genres[data_explore_genres[col]==1]['budget'].median()
total_revenue = data_explore_genres[data_explore_genres[col]==1]['revenue'].sum()
median_revenue = data_explore_genres[data_explore_genres[col]==1]['revenue'].median()
median_popularity = data_explore_genres[data_explore_genres[col]==1]['popularity'].median()
genres_info.append([col, total_movies, total_budget, median_budget, total_revenue, median_revenue, median_popularity] )<feature_engineering>
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test )
genres_info = pd.DataFrame(genres_info, columns=['genres', 'movies_count', 'total_budget', 'median_budget', 'total_revenue', 'median_revenue', 'median_popularity'])
 genres_info['total_budget(million)'] = genres_info['total_budget'].apply(lambda x : x//1000000)
 genres_info['median_budget(million)'] = genres_info['median_budget'].apply(lambda x : x//1000000)
 genres_info['total_revenue(million)'] = genres_info['total_revenue'].apply(lambda x : x//1000000)
 genres_info['median_revenue(million)'] = genres_info['median_revenue'].apply(lambda x : x//1000000)
 genres_info[['genres', 'movies_count', 'total_budget(million)', 'median_budget(million)', 'total_revenue(million)', 'median_revenue(million)', 'median_popularity']]<import_modules>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
for c in ['P_emaildomain', 'R_emaildomain']:
df_train[c + '_bin'] = df_train[c].map(emails)
 df_test[c + '_bin'] = df_test[c].map(emails)
 df_train[c + '_suffix'] = df_train[c].map(lambda x: str(x ).split('.')[-1])
 df_test[c + '_suffix'] = df_test[c].map(lambda x: str(x ).split('.')[-1])
 df_train[c + '_suffix'] = df_train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 df_test[c + '_suffix'] = df_test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
from wordcloud import WordCloud<sort_values>
for f in df_train.drop('isFraud', axis=1 ).columns:
if df_train[f].dtype=='object' or df_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(df_train[f].values)+ list(df_test[f].values))
df_train[f] = lbl.transform(list(df_train[f].values))
df_test[f] = lbl.transform(list(df_test[f].values))
corr_matrix['revenue'].sort_values(ascending=False )<import_modules>
df_train['Trans_min_mean'] = df_train['TransactionAmt'] - df_train['TransactionAmt'].mean()
df_train['Trans_min_std'] = df_train['Trans_min_mean'] / df_train['TransactionAmt'].std()
df_test['Trans_min_mean'] = df_test['TransactionAmt'] - df_test['TransactionAmt'].mean()
df_test['Trans_min_std'] = df_test['Trans_min_mean'] / df_test['TransactionAmt'].std()
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, PowerTransformer, MultiLabelBinarizer<prepare_x_and_y>
df_train['TransactionAmt_to_mean_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_mean_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_std_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('std')
 df_train['TransactionAmt_to_std_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_mean_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_mean_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_std_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_std_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('std' )
X = data.drop(columns=['revenue'], axis=1 ).copy()
y = data['revenue'].copy()
X.shape, y.shape<feature_engineering>
df_train['TransactionAmt'] = np.log(df_train['TransactionAmt'])
 df_test['TransactionAmt'] = np.log(df_test['TransactionAmt'] )
top_30_values = dict()
list_of_genres_names = list(X['genres'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_genres =(Counter([i for j in list_of_genres_names for i in j] ).most_common(30))
top_30_values['genres'] = [x for x, y in top_30_genres]
list_of_production_companies_names = list(X['production_companies'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_production_companies =(Counter([i for j in list_of_production_companies_names for i in j] ).most_common(30))
top_30_values['production_companies'] = [x for x, y in top_30_production_companies]
list_of_production_countries_names = list(X['production_countries'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_production_countries =(Counter([i for j in list_of_production_countries_names for i in j] ).most_common(30))
top_30_values['production_countries'] = [x for x, y in top_30_production_countries]
list_of_keywords = list(X['Keywords'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_keywords =(Counter([i for j in list_of_keywords for i in j] ).most_common(30))
top_30_values['Keywords'] = [x for x, y in top_30_keywords]
list_of_cast_names = list(X['cast'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x if i['order'] in(0, 1, 2)] if x != {} else [] ).values)
 top_30_cast =(Counter([i for j in list_of_cast_names for i in j] ).most_common(30))
top_30_values['cast'] = [x for x, y in top_30_cast]<define_variables>
df_test['isFraud'] = 'test'
df = pd.concat([df_train, df_test], axis=0, sort=False)
 df = df.reset_index()
df = df.drop('index', axis=1 )
drop_cols = ['id', 'homepage', 'imdb_id', 'original_title', 'spoken_languages', 'overview', 'poster_path', 'tagline', 'title', 'crew']
encoded_cols = []<categorify>
def PCA_change(df, cols, n_components, prefix='PCA_', rand_seed=4):
pca = PCA(n_components=n_components, random_state=rand_seed)
 principalComponents = pca.fit_transform(df[cols])
 principalDf = pd.DataFrame(principalComponents)
 df.drop(cols, axis=1, inplace=True)
 principalDf.rename(columns=lambda x: str(prefix)+str(x), inplace=True)
 df = pd.concat([df, principalDf], axis=1)
 return df
class CustomAttr(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
try:
X['is_sequel'] = X['belongs_to_collection'].apply(lambda x: 0 if pd.isna(x)else 1)
 X['release_date']= X['release_date'].apply(lambda x: self.modify_date(x))
X['release_year'] = pd.DatetimeIndex(X['release_date'] ).year
X['release_month'] = pd.DatetimeIndex(X['release_date'] ).month
X['release_day'] = pd.DatetimeIndex(X['release_date'] ).day
X['release_dow'] = pd.DatetimeIndex(X['release_date'] ).dayofweek
X = X.drop(['belongs_to_collection', 'release_date'], axis=1)
 return X
except Exception as e:
print("CustomAttr: Exception caught: {}".format(e))
@staticmethod
def modify_date(x):

 try:
if x is np.nan:
x='01/01/00'
x=str(x)
 year=x.split('/')[2]
if int(year)<20:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
except Exception as e:
print("CustomAttr: modify_date() function - exception caught for date {}: {}".format(x,e))<categorify>
for col in mas_v:
df[col] = df[col].fillna(( df[col].min() - 2))
df[col] =(minmax_scale(df[col], feature_range=(0,1)))
 df = PCA_change(df, mas_v, prefix='PCA_V_', n_components=30 )
class JSONHandler(BaseEstimator, TransformerMixin):
def __init__(self):

 self.mlbs = dict()
def fit(self, X, y=None):
return self
def transform(self, X):
for col in list(X.columns):
try:
X[col] = X[col].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x))
X[col] = X[col].apply(lambda x: self.get_names(x, col))
if not(col in self.mlbs.keys()):
self.mlbs[col] = MultiLabelBinarizer()
X_enc = pd.DataFrame(self.mlbs[col].fit_transform(X[col]),columns=self.mlbs[col].classes_,
index=X.index)
 encoded_cols.extend(list(self.mlbs[col].classes_))
else:
X_enc = pd.DataFrame(self.mlbs[col].transform(X[col]),columns=self.mlbs[col].classes_,
index=X.index)
 X = X.drop(col, axis=1)
 X = pd.concat([X, X_enc], axis=1)
 except Exception as e:
print("JSONHandler: Exception caught for {}: {}".format(col,e))
return X
@staticmethod
def get_names(x, col):

 names = []
try:
names = [item['name'] for item in x if item['name'] in top_30_values[col]]
if len(names)==0:
names.append('other_'+col)
 return names
except Exception as e:
print("JSONHandler: get_names() function - exception caught {}: {}".format(x,e))<categorify>
df = reduce_mem_usage(df )
num_pipeline = Pipeline([('imputer', SimpleImputer(strategy='median')) , 
('scaler', PowerTransformer())])
 cat_pipeline = Pipeline([('imputer', SimpleImputer(strategy='most_frequent')) , 
('cat_enc', OneHotEncoder(handle_unknown='ignore')) ])
 pre_process = ColumnTransformer([('drop_cols', 'drop', drop_cols), 
('num_process', num_pipeline, ['budget', 'popularity', 'runtime']), 
('add_custom_attrs', CustomAttr() , ['belongs_to_collection', 'release_date']), 
('cat_process', cat_pipeline, ['original_language', 'status']), 
('jason_handler', JSONHandler() , ['genres', 'production_companies', 'production_countries', 'Keywords', 'cast'])], remainder='passthrough')
 X_transformed = pre_process.fit_transform(X)
 X_transformed.shape<split>
df_train, df_test = df[df['isFraud'] != 'test'], df[df['isFraud'] == 'test'].drop('isFraud', axis=1 )
X_train_transformed, X_test_transformed, y_train, y_test = train_test_split(X_transformed, np.log1p(y), test_size=0.2, random_state=42)
 X_train_transformed.shape, X_test_transformed.shape<define_variables>
X_train = df_train.sort_values('TransactionDT' ).drop(['isFraud',
'TransactionDT',
],
axis=1)
 y_train = df_train.sort_values('TransactionDT')['isFraud'].astype(bool)
 X_test = df_test.sort_values('TransactionDT' ).drop(['TransactionDT',
],
axis=1)
 del df_train
df_test = df_test[["TransactionDT"]]
feature_columns = ['budget', 'popularity', 'runtime', 'is_sequel', 'release_year', 'release_month', 'release_day', 'release_dow'] + list(pre_process.transformers_[3][1]['cat_enc'].get_feature_names(['original_language', 'status'])) + encoded_cols
len(feature_columns )<choose_model_class>
def objective(params):
time1 = time.time()
params = {
'max_depth': int(params['max_depth']),
'gamma': "{:.3f}".format(params['gamma']),
'subsample': "{:.2f}".format(params['subsample']),
'reg_alpha': "{:.3f}".format(params['reg_alpha']),
'reg_lambda': "{:.3f}".format(params['reg_lambda']),
'learning_rate': "{:.3f}".format(params['learning_rate']),
'num_leaves': '{:.3f}'.format(params['num_leaves']),
'colsample_bytree': '{:.3f}'.format(params['colsample_bytree']),
'min_child_samples': '{:.3f}'.format(params['min_child_samples']),
'feature_fraction': '{:.3f}'.format(params['feature_fraction']),
'bagging_fraction': '{:.3f}'.format(params['bagging_fraction'])
 }
print("
print(f"params = {params}")
 FOLDS = 7
count=1
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=42)
 tss = TimeSeriesSplit(n_splits=FOLDS)
 y_preds = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 score_mean = 0
for tr_idx, val_idx in tss.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=600, random_state=4, verbose=True,
tree_method='gpu_hist',
**params 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 score = make_scorer(roc_auc_score, needs_proba=True )(clf, X_vl, y_vl)
 score_mean += score
print(f'{count} CV - score: {round(score, 4)}')
 count += 1
time2 = time.time() - time1
print(f"Total Time Run: {round(time2 / 60,2)}")
 gc.collect()
print(f'Mean ROC_AUC: {score_mean / FOLDS}')
 del X_tr, X_vl, y_tr, y_vl, clf, score
return -(score_mean / FOLDS)
 space = {
'max_depth': hp.quniform('max_depth', 7, 23, 1),
'reg_alpha': hp.uniform('reg_alpha', 0.01, 0.4),
'reg_lambda': hp.uniform('reg_lambda', 0.01,.4),
'learning_rate': hp.uniform('learning_rate', 0.01, 0.2),
'colsample_bytree': hp.uniform('colsample_bytree', 0.3,.9),
'gamma': hp.uniform('gamma', 0.01,.7),
'num_leaves': hp.choice('num_leaves', list(range(20, 250, 10))),
'min_child_samples': hp.choice('min_child_samples', list(range(100, 250, 10))),
'subsample': hp.choice('subsample', [0.2, 0.4, 0.5, 0.6, 0.7,.8,.9]),
'feature_fraction': hp.uniform('feature_fraction', 0.4,.8),
'bagging_fraction': hp.uniform('bagging_fraction', 0.4,.9)
 }
kf = KFold(n_splits=5, shuffle=True, random_state=42 )<compute_train_metric>
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=27)
 best_params = space_eval(space, best )
results = []
def performance_measures(model, store_results=True):
train_rmses = cross_val_score(model, X_train_transformed, y_train, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 train_rmses *= -1
train_mean_rmse = np.mean(train_rmses)
 test_rmses = cross_val_score(model, X_test_transformed, y_test, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 test_rmses *= -1
test_mean_rmse = np.mean(test_rmses)
 print("Train Mean RMSE: {}
Test Mean RMSE: {}".format(train_mean_rmse, test_mean_rmse))
if store_results:
results.append([model.__class__.__name__, train_mean_rmse, test_mean_rmse] )<train_model>
print("BEST PARAMS: ", best_params)
 best_params['max_depth'] = int(best_params['max_depth'] )
ridge_reg = Ridge(alpha=1, random_state=42)
 ridge_reg.fit(X_train_transformed, y_train )<compute_test_metric>
clf = xgb.XGBClassifier(
 n_estimators=300,
**best_params,
tree_method='gpu_hist' 
)
 clf.fit(X_train, y_train)
 y_preds = clf.predict_proba(X_test)[:,1]
<train_model><EOS> 
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('XGB_hypopt_model.csv' )