Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<load_from_csv>
warnings.filterwarnings('ignore' )
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 y = train_data.Survived
X = train_data.drop(['Survived'], axis = 1)
 X.info()<split>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
X_train_full, X_valid_full, y_train, y_valid = train_test_split(X,y, test_size = 0.2, random_state = 0 )<prepare_x_and_y>
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df
categorical_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and
X_train_full[cname].dtype == "object"]
numerical_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]
my_cols = categorical_cols + numerical_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()<categorify>
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
numerical_transformer = SimpleImputer(strategy='mean')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
<categorify>
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max() -1)].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(
 test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(
 train_df['TransactionID'])].reset_index(drop=True)
 del train_df['DT_M'], test_df['DT_M']
else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl')
 base_columns = list(train_df)+ list(train_identity)
 print('Shape control:', train_df.shape, test_df.shape )
X_train = preprocessor.fit_transform(X_train)
 X_valid = preprocessor.transform(X_valid )<import_modules>
for df in [train_df, test_df]:
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['D9'] = np.where(df['D9'].isna() ,0,1 )
from xgboost import XGBClassifier<train_model>
i_cols = ['card1']
for col in i_cols:
valid_card = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card = valid_card[valid_card>2]
valid_card = list(valid_card.index)
 train_df[col] = np.where(train_df[col].isin(test_df[col]), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(train_df[col]), test_df[col], np.nan)
 train_df[col] = np.where(train_df[col].isin(valid_card), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(valid_card), test_df[col], np.nan )
model = XGBClassifier(n_estimators=200, learning_rate=0.01)
 model.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_valid, y_valid)],
verbose=False )<predict_on_test>
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train_df, test_df]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8)
 df['M_na'] = df[i_cols].isna().sum(axis=1 ).astype(np.int8 )
predictions = model.predict(X_valid )<compute_test_metric>
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )
accuracy_score(y_valid, predictions )<predict_on_test>
train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)
 test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)
 train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['card3'].astype(str)+'_'+train_df['card4'].astype(str)
 test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['card3'].astype(str)+'_'+test_df['card4'].astype(str)
 train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)
 test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)
 train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)
 test_df['TransactionAmt_check'] = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)
 i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df)
 test_df[new_col_name] = test_df[col].map(temp_df)
 train_df['TransactionAmt'] = np.log1p(train_df['TransactionAmt'])
 test_df['TransactionAmt'] = np.log1p(test_df['TransactionAmt'] )
PassengerId = test_data.PassengerId,
test_data = preprocessor.transform(test_data)
 final_predictions = model.predict(test_data )<save_to_csv>
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
for df in [train_df, test_df]:
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
output = pd.DataFrame({'PassengerId': PassengerId[0],
'Survived': final_predictions})
 output.to_csv('submission.csv', index=False )<set_options>
for df in [train_identity, test_identity]:
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['DeviceInfo_device'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['DeviceInfo_version'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_30'] = df['id_30'].fillna('unknown_device' ).str.lower()
df['id_30_device'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['id_30_version'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_31'] = df['id_31'].fillna('unknown_device' ).str.lower()
df['id_31_device'] = df['id_31'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
%matplotlib inline 
<define_variables>
temp_df = train_df[['TransactionID']]
temp_df = temp_df.merge(train_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
train_df = pd.concat([train_df,temp_df], axis=1)
 temp_df = test_df[['TransactionID']]
temp_df = temp_df.merge(test_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
test_df = pd.concat([test_df,temp_df], axis=1 )
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<load_from_csv>
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceInfo_device','DeviceInfo_version',
'id_30','id_30_device','id_30_version',
'id_31_device',
'id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_total'] = train_df[col].map(fq_encode)
 test_df[col+'_total'] = test_df[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train_df[[col,period]], test_df[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train_df[new_column] =(train_df[col].astype(str)+ '_' + train_df[period].astype(str)).map(fq_encode)
 test_df[new_column] =(test_df[col].astype(str)+ '_' + test_df[period].astype(str)).map(fq_encode)
 train_df[new_column] /= train_df[period+'_total']
test_df[new_column] /= test_df[period+'_total']
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
if LOCAL_TEST:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 20000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=2 )
<merge><EOS> 
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )