Kaggle competition: IEEE-CIS Fraud Detection
new_data['homepage_end']=new_data[new_data['homepage'].notna() ]['homepage'].str.findall(r'\.( [a-z]+ )(?:\/|$)' ).apply(lambda x:x[0])
 new_data['homepage_end'].head()<data_type_conversions>
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
new_data['homepage_end']=new_data['homepage_end'].fillna('unknow' )<categorify>
gc.collect()
page=pd.get_dummies(new_data['homepage_end'])
 page.head()<drop_column>
%%time
train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 sample_submission = pd.read_csv(f'{folder_path}sample_submission.csv', index_col='TransactionID')
 df_train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True, on='TransactionID')
 df_test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True, on='TransactionID')
 print(df_train.shape)
 print(df_test.shape)
 del train_transaction, train_identity, test_transaction, test_identity
new_data=new_data.drop('homepage',axis=1 )<drop_column>
%%time
df_train = reduce_mem_usage(df_train)
 df_test = reduce_mem_usage(df_test )
new_data=new_data.drop('poster_path',axis=1 )<feature_engineering>
%%time
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
for c in ['P_emaildomain', 'R_emaildomain']:
df_train[c + '_bin'] = df_train[c].map(emails)
 df_test[c + '_bin'] = df_test[c].map(emails)
 df_train[c + '_suffix'] = df_train[c].map(lambda x: str(x ).split('.')[-1])
 df_test[c + '_suffix'] = df_test[c].map(lambda x: str(x ).split('.')[-1])
 df_train[c + '_suffix'] = df_train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 df_test[c + '_suffix'] = df_test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))
new_data['len_overview'][new_data['len_overview']==0]=1
new_data['len_overview_budget']=new_data['budget']/(new_data['len_overview']+1 )<sort_values>
%%time
for f in df_train.drop('isFraud', axis=1 ).columns:
if df_train[f].dtype=='object' or df_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(df_train[f].values)+ list(df_test[f].values))
df_train[f] = lbl.transform(list(df_train[f].values))
df_test[f] = lbl.transform(list(df_test[f].values))
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()<define_variables>
%%time
df_train = df_train.fillna(-999)
 df_test = df_test.fillna(-999 )
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove<filter>
%%time
y_train_full = df_train['isFraud'].copy()
X_train_full = df_train.drop('isFraud', axis=1)
 X_test = df_test.copy()
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )<groupby>
%%time
X_train, X_val, y_train, Y_val = train_test_split(X_train_full, y_train_full,
test_size=0.15,
random_state=42 )
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )<feature_engineering>
%%time
clf = RandomForestClassifier(n_estimators=1000, max_depth=2, random_state=0, n_jobs=4)
 clf.fit(X_train, y_train)
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()<drop_column>
%%time
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('RandomForest.csv' )
new_data=new_data.drop('genres',axis=1 )<find_best_params>
del X_train, X_val, y_train, Y_val
gc.collect()
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
<feature_engineering>
%%time
df_train['Trans_min_mean'] = df_train['TransactionAmt'] - df_train['TransactionAmt'].mean()
df_train['Trans_min_std'] = df_train['Trans_min_mean'] / df_train['TransactionAmt'].std()
df_test['Trans_min_mean'] = df_test['TransactionAmt'] - df_test['TransactionAmt'].mean()
df_test['Trans_min_std'] = df_test['Trans_min_mean'] / df_test['TransactionAmt'].std()
df_train['TransactionAmt_to_mean_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_mean_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_train['TransactionAmt_to_std_card1'] = df_train['TransactionAmt'] / df_train.groupby(['card1'])['TransactionAmt'].transform('std')
 df_train['TransactionAmt_to_std_card4'] = df_train['TransactionAmt'] / df_train.groupby(['card4'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_mean_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_mean_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('mean')
 df_test['TransactionAmt_to_std_card1'] = df_test['TransactionAmt'] / df_test.groupby(['card1'])['TransactionAmt'].transform('std')
 df_test['TransactionAmt_to_std_card4'] = df_test['TransactionAmt'] / df_test.groupby(['card4'])['TransactionAmt'].transform('std')
 df_train['TransactionAmt'] = np.log(df_train['TransactionAmt'])
 df_test['TransactionAmt'] = np.log(df_test['TransactionAmt'] )
new_data['production_countries']=country 
<feature_engineering>
%%time
df_test['isFraud'] = 'test'
df = pd.concat([df_train, df_test], axis=0, sort=False)
 df = df.reset_index()
df = df.drop('index', axis=1 )
new_data['original_and_new']=new_data['original_title']==new_data['title']<drop_column>
def PCA_change(df, cols, n_components, prefix='PCA_', rand_seed=4):
pca = PCA(n_components=n_components, random_state=rand_seed)
 principalComponents = pca.fit_transform(df[cols])
 principalDf = pd.DataFrame(principalComponents)
 df.drop(cols, axis=1, inplace=True)
 principalDf.rename(columns=lambda x: str(prefix)+str(x), inplace=True)
 df = pd.concat([df, principalDf], axis=1)
 return df
new_data=new_data.drop('original_title',axis=1)
 new_data.head()<concatenate>
%%time
for col in mas_v:
df[col] = df[col].fillna(( df[col].min() - 2))
df[col] =(minmax_scale(df[col], feature_range=(0,1)))
 df = PCA_change(df, mas_v, prefix='PCA_V_', n_components=30)
new_data=pd.concat([new_data,page],axis=1 )<feature_engineering>
%%time
df_train, df_test = df[df['isFraud'] != 'test'], df[df['isFraud'] == 'test'].drop('isFraud', axis=1 )
new_data['production_companies']=new_data['production_companies'].str.findall(r''name'?:\s?'([A-Za-z]+)')
 new_data.fillna('Unknow')
 print('接下来就是地图可视化了' )<drop_column>
%%time
X_train = df_train.sort_values('TransactionDT' ).drop(['isFraud',
'TransactionDT',
],
axis=1)
 y_train = df_train.sort_values('TransactionDT')['isFraud'].astype(bool)
 X_test = df_test.sort_values('TransactionDT' ).drop(['TransactionDT',
],
axis=1)
 del df_train
df_test = df_test[["TransactionDT"]]
new_data=new_data.drop('imdb_id',axis=1 )<feature_engineering>
def objective(params):
time1 = time.time()
params = {
'max_depth': int(params['max_depth']),
'gamma': "{:.3f}".format(params['gamma']),
'subsample': "{:.2f}".format(params['subsample']),
'reg_alpha': "{:.3f}".format(params['reg_alpha']),
'reg_lambda': "{:.3f}".format(params['reg_lambda']),
'learning_rate': "{:.3f}".format(params['learning_rate']),
'num_leaves': '{:.3f}'.format(params['num_leaves']),
'colsample_bytree': '{:.3f}'.format(params['colsample_bytree']),
'min_child_samples': '{:.3f}'.format(params['min_child_samples']),
'feature_fraction': '{:.3f}'.format(params['feature_fraction']),
'bagging_fraction': '{:.3f}'.format(params['bagging_fraction'])
 }
print("
print(f"params = {params}")
 FOLDS = 7
count=1
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=42)
 tss = TimeSeriesSplit(n_splits=FOLDS)
 y_preds = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 score_mean = 0
for tr_idx, val_idx in tss.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=600, random_state=4, verbose=True,
tree_method='gpu_hist',
**params 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 score = make_scorer(roc_auc_score, needs_proba=True )(clf, X_vl, y_vl)
 score_mean += score
print(f'{count} CV - score: {round(score, 4)}')
 count += 1
time2 = time.time() - time1
print(f"Total Time Run: {round(time2 / 60,2)}")
 print(f'Mean ROC_AUC: {score_mean / FOLDS}')
 del X_tr, X_vl, y_tr, y_vl, clf, score
gc.collect()
return -(score_mean / FOLDS)
 space = {
'max_depth': hp.quniform('max_depth', 7, 23, 1),
'reg_alpha': hp.uniform('reg_alpha', 0.01, 0.4),
'reg_lambda': hp.uniform('reg_lambda', 0.01,.4),
'learning_rate': hp.uniform('learning_rate', 0.01, 0.2),
'colsample_bytree': hp.uniform('colsample_bytree', 0.3,.9),
'gamma': hp.uniform('gamma', 0.01,.7),
'num_leaves': hp.choice('num_leaves', list(range(20, 250, 10))),
'min_child_samples': hp.choice('min_child_samples', list(range(100, 250, 10))),
'subsample': hp.choice('subsample', [0.2, 0.4, 0.5, 0.6, 0.7,.8,.9]),
'feature_fraction': hp.uniform('feature_fraction', 0.4,.8),
'bagging_fraction': hp.uniform('bagging_fraction', 0.4,.9)
 }
new_data['spoken_languages']=new_data['spoken_languages'].str.findall(r''([a-z]{2})'' )<data_type_conversions>
gc.collect()
new_data['production_companies']=new_data['production_companies'].fillna('unknow' )<data_type_conversions>
%%time
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=5)
 best_params = space_eval(space, best )
new_data['spoken_languages']=new_data['spoken_languages'].fillna('unknow' )<feature_engineering>
%%time
print("BEST PARAMS: ", best_params)
 best_params['max_depth'] = int(best_params['max_depth'] )
new_data['Keywords']=new_data['Keywords'].str.findall(r''?:\s?'([a-z]+\s?[a-z]+)'' ).fillna('unkonw' )<count_unique_values>
%%time
clf = xgb.XGBClassifier(
 n_estimators=300,
**best_params,
tree_method='gpu_hist'
)
d = Counter([i for j in list_of_keywords for i in j] ).most_common(30)
 d<feature_engineering>
%%time
kfold = KFold(n_splits=10, random_state=7)
 results = cross_val_score(clf, X_train, y_train, cv=kfold)
 print("Accuracy: %.2f%%(%.2f%%)" %(results.mean() *100, results.std() *100))
for i in d:
m=i[0]
new_data['Keywords_'+m]=new_data['Keywords'].apply(lambda x:1 if m in x else 0 )<drop_column>
%%time
clf.fit(X_train, y_train)
 y_preds = clf.predict_proba(X_test)[:,1]
new_data=new_data.drop('Keywords',axis=1 )<data_type_conversions>
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('XGB_hypopt_model.csv' )
new_data['cast']=new_data['cast'].fillna('unknow' )<feature_engineering>
for i in d:
m=i[0][0]
new_data['cast_name_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )<count_values>