Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<save_to_csv>
warnings.filterwarnings('ignore' )
submit = pd.DataFrame({'id': test.id, 'revenue':np.expm1(predict)})
 submit.to_csv('submission.csv', index=False )<categorify>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def fill_budget(in_train, in_test):
def fb_proc_json_len(string):
try:
data = eval(string)
 return len(data)
 except:
return 0
test = in_test.copy()
test['revenue'] = np.nan
total = pd.concat([in_train, test], axis=0)
 fb_features = ['popularity', 'runtime', 'cast', 'crew', 'genre_count',
'company_count', 'release_year', 'release_month']
fb_target = 'budget'
train = total[total.budget > 0]
pred = total[total.budget == 0]
model = GradientBoostingRegressor(criterion='mse', learning_rate=0.15, loss='huber', max_depth=6,
max_features='sqrt', min_samples_leaf=4, min_samples_split=9, n_estimators=110)
 model.fit(train[fb_features], train[fb_target])
 predict = model.predict(pred[fb_features])
 pred[fb_target] = predict
out_train = in_train.copy()
out_test = in_test.copy()
for idx, row in out_train[out_train.budget == 0].iterrows() :
mid = row.id
out_train.loc[out_train.id == mid, 'budget'] = pred.loc[pred.id == mid, 'budget']
for idx, row in out_test[out_test.budget == 0].iterrows() :
mid = row.id
out_test.loc[out_test.id == mid, 'budget'] = pred.loc[pred.id == mid, 'budget']
return(out_train, out_test )<prepare_x_and_y>
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
candidates = train.select_dtypes(include=['int64', 'float64', 'uint8', 'int8'] ).columns.tolist()
candidates.remove('id')
 candidates.remove('revenue')
 X, y = train[candidates], train['revenue']
model = GradientBoostingRegressor(criterion='mse', max_depth=6, max_features='sqrt',
min_samples_leaf=4, min_samples_split=9, n_estimators=110, loss='huber')
 sfs = SFS(estimator=model,
k_features=(3, 9),
forward=True,
floating=False,
scoring='neg_mean_squared_error',
cv=5)
 sfs.fit(X, y, custom_feature_names=candidates)
 print('best combination(ACC: %.3f): %s
' %(sfs.k_score_, sfs.k_feature_idx_))
print('all subsets:
', sfs.subsets_)
 fig = plot_sfs(sfs.get_metric_dict() , kind='std_err')
 plt.title('Sequential Forward Selection(w.StdErr)')
 plt.grid()
plt.show() 
<save_to_csv>
print('Load Data')
 train_df = pd.read_pickle('.. /input/mydata/train_df_with_mn.pkl')
 test_df = pd.read_pickle('.. /input/mydata/test_df_with_mn.pkl')
 remove_features = pd.read_pickle('.. /input/mydata/remove_features_with_mn.pkl')
sfs_fatures = ['budget', 'popularity', 'runtime', 'tagline_len', 'release_year', 'genre_Drama', 'genre_Family', 'genre_Thriller', 'genre_Crime']
model = GradientBoostingRegressor(criterion='friedman_mse', max_depth=5, max_features='sqrt',
min_samples_leaf=4, min_samples_split=3, n_estimators=110, random_state=1)
 model.fit(train[sfs_fatures], train['revenue'])
 predict = model.predict(test[sfs_fatures])
 submit = pd.DataFrame({'id': test.id, 'revenue':np.expm1(predict)})
 submit.to_csv('submission.csv', index=False)
<load_from_csv>
features_columns = [col for col in list(train_df)if col not in remove_features]
if not LOCAL_TEST:
train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df )
train_data = pd.read_csv('.. /input/train.csv', sep=None, engine='python')
 test_data = pd.read_csv('.. /input/test.csv', sep=None, engine='python' )<train_model>
train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']] = train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']].astype(int)
 test_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']] = train_df[['P_emaildomain', 'R_emaildomain', 'DeviceType']].astype(int )
m_train, n1 = train_data.shape
m_test, n2 = test_data.shape
print("{} training samples".format(m_train))
print("{} test samples".format(m_test))
print("{} features to t".format(n1))<categorify>
X_train = train_df[features_columns]
y_train = train_df[TARGET]
X_test = test_df[features_columns]
print("- The target is:
{} ".format([item for item in train_data.columns.tolist() if item not in test_data.columns.tolist() ]))
print(train_data["".join([item for item in train_data.columns.tolist() if item not in test_data.columns.tolist() ])].dtypes )<data_type_conversions>
NFOLDS = 6
folds = GroupKFold(n_splits = NFOLDS)
 y_preds = np.zeros(X_test.shape[0])
 split_groups = train_df['DT_M']
train_data.drop(columns=['homepage', 'imdb_id', 'original_title', 'poster_path'], inplace=True)
 test_data.drop(columns=['homepage', 'imdb_id', 'original_title', 'poster_path'], inplace=True)
 train_data.budget = train_data.budget.astype(float)
 train_data.revenue = train_data.revenue.astype(float)
 test_data.budget = test_data.budget.astype(float )<feature_engineering>
del train_df, test_df
color = {'boxes': 'DarkGreen', 'whiskers': 'DarkOrange','medians': 'DarkBlue', 'caps': 'Gray'}
plt.figure(figsize=(20,10))
belongs_df = train_data[['belongs_to_collection', 'revenue']]
belongs_df['belongs_to_collection_bool'] = True
belongs_df.belongs_to_collection_bool[train_data.belongs_to_collection.isnull() ] = False
belongs_df['in_saga'] = np.nan
belongs_df.in_saga[belongs_df.belongs_to_collection_bool == True] = belongs_df.revenue[belongs_df.belongs_to_collection_bool == True]
belongs_df['no_saga'] = np.nan
belongs_df.no_saga[belongs_df.belongs_to_collection_bool == False] = belongs_df.revenue[belongs_df.belongs_to_collection_bool == False]
belongs_df.belongs_to_collection[~train_data.belongs_to_collection.isnull() ] = train_data.belongs_to_collection[~train_data.belongs_to_collection.isnull() ].str.split("'name': '" ).str[1].str.split("', 'poster" ).str[0]
belongs_df.drop(columns=['revenue', 'belongs_to_collection', 'belongs_to_collection_bool'], inplace=True)
 print("Movies in saga: {}/{}={}% ".format(len(~belongs_df.in_saga.isnull()), len(belongs_df), 100.0*len(~belongs_df.in_saga.isnull())/len(belongs_df)))
 sns.boxplot(data=belongs_df, orient='h')
 plt.xlabel('value')
 plt.ylabel('Data')
 plt.title('Box plot of Belongig to Collection')
 plt.show()<feature_engineering>
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train, groups=split_groups)) :
clf = xgb.XGBClassifier(
 n_estimators=512,
max_depth=16,
learning_rate=0.014,
subsample=0.85,
colsample_bytree=0.85,
tree_method='gpu_hist',
reg_alpha=0.3,
reg_lamdba=0.243 
)
 X_tr, X_vl = X_train.iloc[trn_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 del X_tr, X_vl, y_tr, y_vl
y_preds[:int(len(X_test)/2)] += clf.predict_proba(X_test[:int(len(X_test)/2)])[:,1] / NFOLDS
y_preds[int(len(X_test)/2):] += clf.predict_proba(X_test[int(len(X_test)/2):])[:,1] / NFOLDS
gc.collect()
genres_df = train_data[['genres', 'revenue']]
genres_df['genres_test'] = genres_df['genres']
genres_df.genres_test = genres_df.genres_test.str.strip('[]')
 genres_df.genres_test[genres_df.genres_test.isnull() ] = ''
genres_list = pd.Series(list(set(", ".join(genres_df.genres_test.unique().tolist() ).split('}, ')))).str.split("'name': '" ).str[1].str.split("'" ).str[0].tolist()
for i, genre in enumerate(genres_list):
genres_df[genre] = np.nan
genres_df[genre][genres_df.genres_test.str.contains(genre)] = genres_df.revenue[genres_df.genres_test.str.contains(genre)]
genres_df.drop(columns=['revenue', 'genres', 'genres_test'], inplace=True)
 plt.figure(figsize=(20,12))
sns.boxplot(data=genres_df)
 plt.xlabel('Data')
 plt.ylabel('Value')
 plt.title('Box plot of genres')
 plt.show()<count_duplicates>
if not LOCAL_TEST:
submission = pd.read_csv(".. /input/ieee-fraud-detection/sample_submission.csv")
 submission['isFraud'] = y_preds
submission.to_csv('submission_xgb_groupkfold_final.csv', index=False )