Kaggle competition: IEEE-CIS Fraud Detection
class Config(dict):
def __init__(self, **kwargs):
super().__init__(**kwargs)
 for k, v in kwargs.items() :
setattr(self, k, v)
 def set(self, key, val):
self[key] = val
setattr(self, key, val)
 config = Config(
 testing=False,
bert_model_name="bert-base-uncased",
max_lr=3e-5,
epochs=4,
use_fp16=True,
bs=32,
discriminative=False,
max_seq_len=256,
)<load_pretrained>
train_identity=pd.read_csv('.. /input/train_identity.csv')
 test_identity=pd.read_csv('.. /input/test_identity.csv')
 train_transaction=pd.read_csv('.. /input/train_transaction.csv')
 test_transaction=pd.read_csv('.. /input/test_transaction.csv' )
bert_tok = BertTokenizer.from_pretrained(
 config.bert_model_name,
)<categorify>
train=pd.merge(train_transaction,train_identity,how='left',on='TransactionID')
 test=pd.merge(test_transaction,test_identity,how='left',on='TransactionID' )
def _join_texts(texts:Collection[str], mark_fields:bool=False, sos_token:Optional[str]=BOS):

 if not isinstance(texts, np.ndarray): texts = np.array(texts)
 if is1d(texts): texts = texts[:,None]
df = pd.DataFrame({i:texts[:,i] for i in range(texts.shape[1])})
 text_col = f'{FLD} {1} ' + df[0].astype(str)if mark_fields else df[0].astype(str)
 if sos_token is not None: text_col = f"{sos_token} " + text_col
for i in range(1,len(df.columns)) :
text_col +=(f' {FLD} {i+1} ' if mark_fields else ' ')+ df[i].astype(str)
 return text_col.values<string_transform>
del train_identity,test_identity,train_transaction,test_transaction
class FastAiBertTokenizer(BaseTokenizer):

 def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs):
self._pretrained_tokenizer = tokenizer
self.max_seq_len = max_seq_len
def __call__(self, *args, **kwargs):
return self
def tokenizer(self, t:str)-> List[str]:

 return ["[CLS]"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + ["[SEP]"]<load_from_csv>
DATA_ROOT = Path(".. ")/ "input"
train, test = [pd.read_csv(DATA_ROOT / fname)for fname in ["train.csv", "test.csv"]]
val = train<choose_model_class>
card_cols=['card1','card2','card3','card4','card5','card6']
for c in card_cols:
print(f'Number of unique variables in {c}: ',train[c].nunique() )
fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), pre_rules=[], post_rules=[] )<create_dataframe>
addr=['addr1','addr2','dist1','dist2','P_emaildomain','R_emaildomain']
for c in addr:
print(f'Number of unique variables in {c}: ',train[c].nunique() )
label_cols = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"] 
<choose_model_class>
C_columns=[col for col in train.columns if 'C'==col[0]]
for c in C_columns:
print(f'Number of unique entries in {c}:',train[c].nunique() )
class BertTokenizeProcessor(TokenizeProcessor):
def __init__(self, tokenizer):
super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)
 class BertNumericalizeProcessor(NumericalizeProcessor):
def __init__(self, *args, **kwargs):
super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())) , **kwargs)
 def get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):

 return [BertTokenizeProcessor(tokenizer=tokenizer),
NumericalizeProcessor(vocab=vocab)]<create_dataframe>
D_cols=[col for col in train.columns if col[0]=='D']
for d in D_cols:
print(f'Number of unique entries in {d}:',train[d].nunique() )
class BertDataBunch(TextDataBunch):
@classmethod
def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,
tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,
label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs)-> DataBunch:
"Create a `TextDataBunch` from DataFrames."
p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)
 processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)
 if classes is None and is_listy(label_cols)and len(label_cols)> 1: classes = label_cols
src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)
 if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))
return src.databunch(**kwargs )<create_dataframe>
print(train['D9'].value_counts())
 print('*'*100)
 print(test['D9'].value_counts() )
databunch = BertDataBunch.from_df(".", train, val, test,
tokenizer=fastai_tokenizer,
vocab=fastai_bert_vocab,
text_cols="comment_text",
label_cols=label_cols,
bs=config.bs,
collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),
)<load_pretrained>
M_cols=[col for col in train.columns if col[0]=='M']
for m in M_cols:
print(f'Number of unique entries in {m}:',train[m].nunique() )
bert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=6 )<choose_model_class>
V_cols=[col for col in train.columns if col[0]=='V']
V_type=[train[col].dtype for col in V_cols]
print(set(V_type))
print(len(V_cols))
loss_func = nn.BCEWithLogitsLoss()<choose_model_class>
unique=[(col,train[col].nunique())for col in V_cols]
sorted_unique=sorted(unique,key=lambda x: x[1])
 sorted_unique[:50]
learner = Learner(
 databunch, bert_model,
loss_func=loss_func, 
)
 if config.use_fp16: learner = learner.to_fp16()<find_best_params>
i_cols=[col for col in train.columns if col[0]=='i']
unique_val=[(col,train[col].nunique())for col in i_cols]
unique_val
learner.lr_find()<train_model>
drop_cols=['dist2','C8','C9','C10','D6','D12','V27','V28','V45','V70','V71','V77','V78','V86','V87','V89','V91','V92','V95','V96','V97',
'V101','V102','V103','V107','V126','V127','V128','V130','V131','V132','V133','V134','V135','V136','V137','V143','V145','V150',
'V159','V160','V164','V165','V166','V167','V168','V171','V176','V177','V178','V179','V181','V182','V183','V190','V199','V202',
'V203','V204','V207','V211','V212','V213','V214','V215','V216','V221','V226','V227','V228','V230','V234','V240','V241','V245',
'V246','V255','V256','V257','V258','V259','V261','V263','V264','V265','V270','V271','V272','V273','V274','V275','V276','V277',
'V278','V279','V280','V291','V292','V293','V294','V295','V297','V306','V307','V308','V310','V312','V316','V317','V318','V319',
'V320','V321','V322','V323','V324','V331','V332','V333','V338','V339','TransactionDT','TransactionID']
train=train.drop(drop_cols,1)
 test=test.drop(drop_cols,1)
 train['TransactionAmt']=np.log(train['TransactionAmt'])
 test['TransactionAmt']=np.log(test['TransactionAmt'] )
learner.fit_one_cycle(config.epochs, max_lr=config.max_lr )<predict_on_test>
def get_preds_as_nparray(ds_type)-> np.ndarray:

 preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()
sampler = [i for i in databunch.dl(ds_type ).sampler]
reverse_sampler = np.argsort(sampler)
 return preds[reverse_sampler, :]<predict_on_test>
train=reduce_mem_usage(train)
 test=reduce_mem_usage(test )
test_preds = get_preds_as_nparray(DatasetType.Test )<save_to_csv>
sample_submission = pd.read_csv(DATA_ROOT / "sample_submission.csv")
 if config.testing: sample_submission = sample_submission.head(test.shape[0])
 sample_submission[label_cols] = test_preds
sample_submission.to_csv("predictions.csv", index=False )<set_options>
sns.set(style="darkgrid")
 sns.set(font_scale=1.3)
 warnings.filterwarnings('ignore')
 np.random.seed(37)
 color = sns.color_palette() 
<define_variables>
num_cols=[col for col in train.columns if(( 'float' in str(train[col].dtype)) or('int' in str(train[col].dtype)))]
num_cols.remove('isFraud')
 train['mean']=train[num_cols].mean(axis=1)
 test['mean']=test[num_cols].mean(axis=1)
 train['std']=train[num_cols].std(axis=1)
 test['std']=test[num_cols].std(axis=1)
 train['max']=train[num_cols].max(axis=1)
 test['max']=test[num_cols].max(axis=1)
 train['min']=train[num_cols].min(axis=1)
 test['min']=test[num_cols].min(axis=1)
 train['median']=train[num_cols].median(axis=1)
 test['median']=test[num_cols].median(axis=1)
 train['skew']=train[num_cols].skew(axis=1)
 test['skew']=test[num_cols].skew(axis=1)
 train['kurt']=train[num_cols].kurt(axis=1)
 test['kurt']=test[num_cols].kurt(axis=1 )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'<define_variables>
embed_size = 50
max_features = 20000
maxlen = 100<load_from_csv>
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values<string_transform>
object_cols=[col for col in train.columns if(( 'category' in str(train[col].dtype)) or('object' in str(train[col].dtype)))]
le=LabelEncoder()
for col in object_cols:
le.fit(list(train[col].values)+list(test[col].values))
train[col]=le.transform(list(train[col].values))
test[col]=le.transform(list(test[col].values))
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train)) 
<load_pretrained>
target=train['isFraud']
train=train.drop('isFraud',1 )
file_Name = "testfile"
fileObject = open(file_Name,'wb')
 pickle.dump(tokenizer,fileObject )<load_from_csv>
lrSubmitY= pd.read_csv('.. /input/aggr-dataset/lrSubmitY.csv',index_col=False)
 lrSubmitY.drop(['Unnamed: 0'], inplace=True,axis=1 )<load_from_csv>
nfolds=10
xgb_params=dict(n_estimators=1000,
verbosity=0,
tree_method='gpu_hist',
random_state=seed,
colsample_bytree=0.6,
subsample=0.6,
learning_rate=0.05,
max_depth=9)
 lgb_params=dict(objective='binary',
num_leaves=62,
seed=seed,
max_depth=9,
pos_bagging_fraction=0.5,
neg_bagging_fraction=1.0,
bagging_freq=5,
feature_fraction=0.9,
metric='auc',
learning_rate=0.05,
verbosity=-1,
device='gpu')
 skfold=StratifiedKFold(nfolds,random_state=seed)
 def build_model(params,model='xgb',plot_feature_importance=True):
oof=np.zeros(train.shape[0])
 pred=np.zeros(test.shape[0])
 scores=[]
feature_importance=pd.DataFrame()
for i,(train_index,val_index)in enumerate(skfold.split(train,target)) :
print('Fold :',i+1)
 if model=='xgb':
train_X,val_X=train.iloc[train_index,:],train.iloc[val_index,:]
train_y,val_y=target[train_index],target[val_index]
clf=XGBClassifier(**params)
 clf.fit(train_X,train_y,eval_metric='auc',eval_set=[(val_X,val_y)],early_stopping_rounds=10,verbose=20)
 val_pred=clf.predict_proba(val_X)[:,1]
if model=='lgb':
train_d=lgb.Dataset(train.iloc[train_index,:].values,label=target[train_index].values)
 val_d=lgb.Dataset(train.iloc[val_index,:].values,label=target[val_index].values)
 clf=lgb.train(params,train_d,num_boost_round=1000,valid_sets=[val_d],verbose_eval=20,early_stopping_rounds=10)
 val_pred=clf.predict(train.iloc[val_index,:].values)
 oof[val_index]=val_pred
val_score=roc_auc_score(target[val_index],val_pred)
 scores.append(val_score)
 print(f'Validation score using {model} for fold {i} :'+ str(val_score))
print('-'*100)
 if model=='xgb':
pred+=clf.predict_proba(test)[:,1]/nfolds
if model=='lgb':
pred+=clf.predict(test.values)/nfolds
if model=='xgb':
del train_X,val_X,train_y,val_y
if model=='lgb':
del train_d,val_d
gc.collect()
fold_importance=pd.DataFrame()
fold_importance['feature']=train.columns
if model=='xgb':
fold_importance['importance']=clf.feature_importances_
if model=='lgb':
fold_importance['importance']=clf.feature_importance()
fold_importance['fold']=i+1
feature_importance=pd.concat([feature_importance,fold_importance],axis=0)
 print('Mean validation score :',np.mean(scores))
if plot_feature_importance:
df=feature_importance[['feature','importance']].groupby('feature' ).mean().sort_values(by='importance',ascending=False ).reset_index()
plt.figure(figsize=(10,10))
sns.barplot(x='importance',y='feature',data=df.iloc[:25,:])
 plt.title('Feature Importances')
 return pred
lgbSubmitY= pd.read_csv('.. /input/dataset2/lgbSubmitY.csv',index_col=False)
 lgbSubmitY.drop(['Unnamed: 0'], inplace=True,axis=1 )<load_from_csv>
pred=build_model(xgb_params,model='xgb' )
<define_variables><EOS> 
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub['isFraud']=pred
sub.to_csv('submission.csv',index=False )