Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<categorify>
path_dir = '.. /input/ieee-fraud-detection/'
file_list = os.listdir(path_dir)
 file_list
dummies_month = pd.get_dummies(test['month'], prefix= 'month')
 dummies_season = pd.get_dummies(test['season'], prefix= 'season')
 dummies_weather = pd.get_dummies(test['weather'], prefix= 'weather')
 dummies_year = pd.get_dummies(test['year'], prefix= 'year')
 data_test=pd.concat([test,dummies_month,dummies_season,dummies_weather,dummies_year],axis=1)
 datetimecol = test['datetime']
yLabels=data['count']
dropFeatures = ['datetime' , 'date' ,'season',
'weather','month','year']
dataTest = data_test.drop(dropFeatures,axis=1 )<predict_on_test>
train_identity = pd.read_csv(path_dir+'train_identity.csv')
 train_transaction = pd.read_csv(path_dir+'train_transaction.csv')
 test_identity = pd.read_csv(path_dir+'test_identity.csv')
 test_transaction = pd.read_csv(path_dir+'test_transaction.csv' )
predsTest= rfModel_re.predict(X = dataTest)+ rfModel_ca.predict(X = dataTest)
 submission=pd.DataFrame({'datetime':datetimecol , 'count':[max(0,x)for x in predsTest]} )<save_to_csv>
train_merge = pd.merge(train_identity, train_transaction, on=['TransactionID'], how='right')
 test_merge = pd.merge(test_identity, test_transaction, on=['TransactionID'], how='right')
 train_merge = train_merge.drop(['DeviceInfo'], axis=1)
 test_merge = test_merge.drop(['DeviceInfo'], axis=1)
 del train_identity, train_transaction, test_identity, test_transaction
submission.to_csv('sampleSubmission.csv',index=False )<load_from_csv>
check_null = train_merge.isna().sum() / len(train_merge)
 check_null[check_null >= 0.77]
train = pd.read_csv(".. /input/labeledTrainData.tsv", header = 0, delimiter = '\t')
 def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review, features="html5lib" ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
meaningful_words = [w for w in words if not w in stops]
element = " ".join(meaningful_words)
 return(element)
 num_reviews = train["review"].size
clean_train_reviews = []
def fn(raw_review):
element = review_to_words(raw_review)
 return(element)
 print("Cleaning the Reviews...")
 pool = Pool()
clean_train_reviews = pool.map(fn,train["review"])
 pool.close()
pool.join()
print("The length of the Cleaned Reviews is "+str(len(clean_train_reviews)) )<feature_engineering>
remove_cols = list(check_null[check_null >= 0.77].keys())
 not_remove_cols = ['id_30','id_31','R_emaildomain','P_emaildomain','ProductCD','card4','card6','DeviceType']
for col in not_remove_cols:
if col in remove_cols:
remove_cols.remove(col)
 else:
pass
train_merge = train_merge.drop(remove_cols, axis=1)
 test_merge = test_merge.drop(remove_cols, axis=1)
 print(train_merge.shape)
 print(test_merge.shape )
vectorizer = CountVectorizer(analyzer = "word", \
tokenizer = None, \
preprocessor = None, \
stop_words = None, \
max_features = 1500)
 train_data_features = vectorizer.fit_transform(clean_train_reviews)
 print("Creating Features...")
 train_data_features = train_data_features.toarray()
print("Features Created" )<train_model>
train_merge[object_cols] = train_merge[object_cols].fillna('NaN')
 train_merge[nobject_cols] = train_merge[nobject_cols].fillna(0)
 test_merge[object_cols] = test_merge[object_cols].fillna('NaN')
 nobject_cols = list(nobject_cols)
 nobject_cols.remove('isFraud')
 test_merge[nobject_cols] = test_merge[nobject_cols].fillna(0 )
print("Going to Train the Data Features")
 print("The length of the array is "+str(len(train_data_features)) )<train_model>
def simplify_categorical(data):
mobile_os = [i.split(' ')[0] for i in data['id_30']]
browser = []
for i in data['id_31']:
if i.split(' ')[0] == 'mobile' and len(i.split(' ')) > 1:
browser.append(i.split(' ')[1])
 elif i.split('/')[0] == 'Samsung':
browser.append('samsung')
 elif i.split('/')[0] == 'Microsoft':
browser.append('ie')
 elif i.split('/')[0] == 'Mozilla':
browser.append('firefox')
 elif i.split('/')[0] == 'Generic':
browser.append('android')
 elif len(i.split(' ')) >= 1:
browser.append(i.split(' ')[0])
 else:
browser.append(i)
 p_emaildomain = [i.split('.')[0] for i in data['P_emaildomain']]
r_emaildomain = [i.split('.')[0] for i in data['R_emaildomain']]
data['id_30'] = [i.lower() for i in mobile_os]
data['id_31'] = [i.lower() for i in browser]
data['P_emaildomain'] = [i.lower() for i in p_emaildomain]
data['R_emaildomain'] = [i.lower() for i in r_emaildomain]
return data
train = simplify_categorical(train_merge)
 train_target = train['isFraud']
train = train.drop('isFraud', axis=1)
 test = simplify_categorical(test_merge)
 del train_merge, test_merge
clf = SVC(gamma='auto',cache_size=12000,max_iter=-1)
 print("Training the data set...")
 clf = clf.fit(train_data_features, train["sentiment"])
 print("Training Completed" )<load_from_csv>
categorical_features = list(object_cols)
 for col in categorical_features:
le = LabelEncoder()
le.fit(list(train[col].values)+ list(test[col].values))
train[col] = le.transform(list(train[col].values))
test[col] = le.transform(list(test[col].values))
test = pd.read_csv(".. /input/testData.tsv", header=0, delimiter="\t", \
quoting=3)
 print("Reading the Test Data...")
 num_reviews = len(test["review"])
 clean_test_reviews = []
print("Total number of reviews is "+str(num_reviews))
pool = Pool()
clean_test_reviews = pool.map(fn,test["review"])
 pool.close()
pool.join()
print("Cleaning of the Test Reviews is Completed!!")
 print("The length of the cleaned test reviews is "+str(len(clean_test_reviews)))
<save_to_csv>
train = train.drop('TransactionID', axis=1)
 test_tid = test['TransactionID']
test = test.drop('TransactionID', axis=1 )
test_data_features = vectorizer.transform(clean_test_reviews)
 print("Making the features from the Test Reviews...")
 test_data_features = test_data_features.toarray()
print("Going to predict the Test Features...")
 result = clf.predict(test_data_features)
 print("Prediction Completed!!")
 output = pd.DataFrame(data={"id":test["id"], "sentiment":result})
 print("Saving the Predictions to the File")
 output.to_csv("results.csv", index=False, quoting=3 )<set_options>
train_scale_ = train.drop(categorical_features, axis=1)
 test_scale_ = test.drop(categorical_features, axis=1)
 train_scale =(train_scale_ - train_scale_.mean())/ train_scale_.std()
test_scale =(test_scale_ - train_scale_.mean())/ train_scale_.std()
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload<define_variables>
train[train_scale.columns] = train_scale
test[test_scale.columns] = test_scale
del train_scale_, test_scale_, train_scale, test_scale
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96<load_from_csv>
pca = PCA(n_components=3)
 pca.fit(train)
 pca_train = pca.transform(train)
 pca_test = pca.transform(test )
df_trn=pd.read_csv(train_lbl )<feature_engineering>
train['pca_col1'] = pca_train[:, 0]
train['pca_col2'] = pca_train[:, 1]
train['pca_col3'] = pca_train[:, 2]
test['pca_col1'] = pca_test[:, 0]
test['pca_col2'] = pca_test[:, 1]
test['pca_col3'] = pca_test[:, 2]
del pca_train, pca_test
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)<load_from_csv>
x_train, x_val, y_train, y_val = train_test_split(train, train_target, test_size=0.3, random_state=42 )
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )<define_variables>
lgb_train = lgb.Dataset(x_train, y_train, categorical_feature=categorical_features)
 lgb_val = lgb.Dataset(x_val, y_val, categorical_feature=categorical_features)
 params = {
'objective':'binary',
'boosting_type': 'gbdt',
'max_depth': -1,
'learning_rate': 0.03,
'num_leaves': 500,
'feature_fraction':0.7,
'bagging_fraction':0.8,
'bagging_seed':11,
'n_jobs':-1,
'verbosity': -1
}
lgb_model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_val],
num_boost_round=3000,
early_stopping_rounds=500,
verbose_eval=100 )
data.show_batch(rows=5, figsize=(12,9))<import_modules>
val_preds_lgb = [1 if i>=0.5 else 0 for i in lgb_model.predict(x_val)]
print('Accuracy : {:.2f}'.format(accuracy_score(val_preds_lgb, y_val)))
 print('Recall : {:.2f}'.format(recall_score(val_preds_lgb, y_val)))
 print(classification_report(val_preds_lgb, y_val))
from sklearn.metrics import roc_auc_score<compute_test_metric>
test_preds = lgb_model.predict(test )
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<import_modules>
test_pred_df = pd.DataFrame()
test_pred_df['TransactionID'] = test_tid
test_pred_df['isFraud'] = test_preds
<choose_model_class><EOS> 
test_pred_df.to_csv('lgb_submission.csv', index=False )