Kaggle competition: IEEE-CIS Fraud Detection
testing_data = []
i = 0
def create_testing_data() :
for img in os.listdir(test_directory):
img_array = cv2.imread(os.path.join(test_directory,img),cv2.IMREAD_GRAYSCALE)
 new_img = cv2.resize(img_array,(img_size2,img_size1))
ret, bin_img = cv2.threshold(new_img, 128, 255, cv2.THRESH_BINARY)
 testing_data.append([img,
bin_img] )<train_model>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 del train_transaction, train_identity
gc.collect()
for img in os.listdir(test_directory):
img_array = cv2.imread(os.path.join(test_directory,img),cv2.IMREAD_GRAYSCALE)
 new_img = cv2.resize(img_array,(img_size2,img_size1))
testing_data.append([img,
new_img])
 print('test_directory :', test_directory)
 print('img :', img)
 print('img_array:', img_array)
 print('img_array_shape:', img_array.shape)
 print('new_img :', new_img)
 print('new_img_shape:', new_img.shape)
 break<define_variables>
%%time
test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 del test_transaction, test_identity
gc.collect()
testing_data = []
i = 0<train_model>
selected_features = [
'TransactionAmt', 'ProductCD',
'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13', 'C14',
'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15',
'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9',
'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12',
'V13', 'V17', 'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36',
'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48', 'V49', 'V51',
'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63',
'V64', 'V69', 'V70', 'V71', 'V72', 'V73', 'V74', 'V75', 'V76', 'V78',
'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128',
'V130', 'V131', 'V138', 'V139', 'V140', 'V143', 'V145', 'V146', 'V147', 'V149',
'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161', 'V162',
'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173',
'V175', 'V176', 'V177', 'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189',
'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204', 'V205', 'V206', 'V207',
'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219',
'V220', 'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229',
'V231', 'V233', 'V234', 'V238', 'V239', 'V242', 'V243', 'V244', 'V245', 'V246',
'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261', 'V262',
'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273',
'V274', 'V275', 'V276', 'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285',
'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303', 'V304', 'V306', 'V307',
'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324',
'V326', 'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338',
'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09', 'id_11', 'id_12', 'id_13', 'id_14',
'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33', 'id_36', 'id_37',
'id_38', 'DeviceType', 'DeviceInfo'
]
len(selected_features )
start = time.time()
create_training_data()
print('Elapsed_time: ', time.time() -start, '[sec]' )<count_duplicates>
columns_to_drop = list(set(train.columns)- set(selected_features)- set(['isFraud', 'TransactionDT']))
len(columns_to_drop )
random.shuffle(training_data )<define_variables>
train = train.fillna(-999)
 test = test.fillna(-999 )
x = []
y = []<prepare_x_and_y>
many_same_values_columns_train = [c for c in train.columns if train[c].value_counts(normalize=True ).values[0] > 0.9]
many_same_values_columns_test = [c for c in test.columns if test[c].value_counts(normalize=True ).values[0] > 0.9]
columns_to_drop = list(set(many_same_values_columns_train + many_same_values_columns_test))
columns_to_drop.remove('isFraud')
 len(columns_to_drop )
for features, label in training_data:
x.append(features)
 y.append(label )<define_variables>
os = ['Windows', 'iOS', 'Android', 'Mac OS', 'Linux']
browser = ['chrome', 'mobile safari', 'ie', 'safari', 'edge', 'firefox']
device = ['Windows', 'iOS', 'MacOS', 'SM', 'SAMSUNG', 'Moto', 'LG']
def to_pattern(x: str, patterns):
for p in patterns:
t = re.compile('^(' + p + ' ).*')
 if t.match(x):
return p
return 'other'
def make_os_feature(df):
return df['id_30'].map(lambda x: to_pattern(str(x), os))
def make_browser_feature(df):
return df['id_31'].map(lambda x: to_pattern(str(x), browser))
def make_device_feature(df):
return df['DeviceInfo'].map(lambda x: to_pattern(str(x), device))
print('features: ', x[0])
 print('label : ', y[0] )<split>
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int )
x_train,x_test,y_train,y_test = train_test_split(X,y,test_size=0.3,random_state=50 )<categorify>
def make_aggregates(df, feature_to_group_by, feature):
out = pd.DataFrame(index=df.index)
 agg = df[df[feature] != -999].groupby([feature_to_group_by])[feature]
new_feature = feature + '_' + feature_to_group_by
out[new_feature + '_mean'] = df[feature] / agg.transform('mean')
 out[new_feature + '_std' ] = df[feature] / agg.transform('std')
 return out
def merge_aggregates(df, feature_to_group_by, feature):
return df.merge(make_aggregates(df, feature_to_group_by, feature), how='left', left_index=True, right_index=True)
Y_train = np_utils.to_categorical(y_train,num_classes=10)
 Y_test = np_utils.to_categorical(y_test,num_classes=10 )<choose_model_class>
def make_count_1(feature, df):
temp = df[feature].value_counts(dropna=False)
 new_feature = feature + '_count'
df[new_feature] = df[feature].map(temp)
 def make_count_2(feature, train, test):
temp = pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False)
 new_feature = feature + '_count'
train[new_feature] = train[feature].map(temp)
 test[new_feature] = test[feature].map(temp)
 for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
make_count_1(feature, train)
 make_count_1(feature, test)
 for feature in ['card1', 'id_36']:
make_count_2(feature, train, test )
model = Sequential()<choose_model_class>
def features_interaction(df, feature_1, feature_2):
return df[feature_1].astype(str)+ '_' + df[feature_2].astype(str)
 features_interactions = [
'id_02__id_20',
'id_02__D8',
'D11__DeviceInfo',
'DeviceInfo__P_emaildomain',
'P_emaildomain__C2',
'card2__dist1',
'card1__card5',
'card2__id_20',
'card5__P_emaildomain',
'addr1__card1'
]
for new_feature in features_interactions:
feature_1, feature_2 = new_feature.split('__')
 train[new_feature] = features_interaction(train, feature_1, feature_2)
 test[new_feature] = features_interaction(test, feature_1, feature_2 )
model.add(Conv2D(32,kernel_size=(3,3),activation='relu',input_shape=(img_size1,img_size2,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=(3,3),activation='relu',padding='same'))
model.add(BatchNormalization(axis = 3))
model.add(MaxPooling2D(pool_size=(2,2),padding='same'))
model.add(Dropout(0.3))<choose_model_class>
import matplotlib.pyplot as plt
model.add(Conv2D(64,kernel_size=(3,3),activation='relu',padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=(3,3),activation='relu',padding='same'))
model.add(BatchNormalization(axis = 3))
model.add(MaxPooling2D(pool_size=(2,2),padding='same'))
model.add(Dropout(0.3))<choose_model_class>
def make_day_feature(df, offset=0.58, tname='TransactionDT'):

 days = df[tname] /(3600 * 24)
 encoded_days = np.floor(days - 1 + offset)% 7
return encoded_days
def make_hour_feature(df, tname='TransactionDT'):

 hours = df[tname] /(3600)
 encoded_hours = np.floor(hours)% 24
return encoded_hours
model.add(Conv2D(128,kernel_size=(3,3),activation='relu',padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(128,kernel_size=(3,3),activation='relu',padding='same'))
model.add(BatchNormalization(axis = 3))
model.add(MaxPooling2D(pool_size=(2,2),padding='same'))
model.add(Dropout(0.5))<choose_model_class>
train['Weekday'] = make_day_feature(train )
model.add(Flatten())
 model.add(Dense(units = 512,activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(units = 128,activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(10,activation='softmax'))<choose_model_class>
train['Hour'] = make_hour_feature(train )
model.compile(loss='categorical_crossentropy',metrics=['accuracy'],optimizer='adam' )<choose_model_class>
test['Weekday'] = make_day_feature(test)
 test['Hour'] = make_hour_feature(test )
callbacks = [EarlyStopping(monitor='val_acc',patience=5)]<define_variables>
email_vendors = {
'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple',
-999: 'undefined'
}
us_emails = ['gmail', 'net', 'edu']
def transform_email(df, column):
out = pd.DataFrame(index=df.index)
 out[column + '_vendor'] = df[column].map(email_vendors)
 out[column + '_suffix'] = df[column].map(lambda x: str(x ).split('.')[-1])
 out[column + '_us'] = out[column + '_suffix'].map(lambda x: 'us' if str(x)in us_emails else 'other')
 return out
batch_size = 50
n_epochs = 20<train_model>
results = model.fit(x_train,Y_train,batch_size=batch_size,epochs=n_epochs,verbose=1,validation_data=(x_test,Y_test),callbacks=callbacks )<predict_on_test>
preds = model.predict(np.array(testing_data[0][1] ).reshape(-1,img_size2,img_size1,1))<load_pretrained>
train = train.sort_values('TransactionDT' ).drop('TransactionDT', axis=1)
 test = test.sort_values('TransactionDT' ).drop('TransactionDT', axis=1 )
model.save_weights('./driverdistraction_lr_weights.h5', overwrite=True )<save_model>
def encode_categorial_features_fit(df, columns_to_encode):
encoders = {}
for c in columns_to_encode:
if c in df.columns:
encoder = LabelEncoder()
encoder.fit(df[c].astype(str ).values)
 encoders[c] = encoder
return encoders
def encode_categorial_features_transform(df, encoders):
out = pd.DataFrame(index=df.index)
 for c in encoders.keys() :
if c in df.columns:
out[c] = encoders[c].transform(df[c].astype(str ).values)
 return out
categorial_features_columns = [
'id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21',
'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29', 'id_30', 'id_31',
'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38',
'DeviceType', 'DeviceInfo', 'ProductCD', 'P_emaildomain', 'R_emaildomain',
'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2',
'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9',
'P_emaildomain_vendor', 'P_emaildomain_suffix', 'P_emaildomain_us',
'R_emaildomain_vendor', 'R_emaildomain_suffix', 'R_emaildomain_us'
] + features_interactions
categorial_features_encoders = encode_categorial_features_fit(
 pd.concat([train, test], join='outer', sort=False),
categorial_features_columns )
model.save('./driverdistraction_lr_weights.h5' )<load_pretrained>
temp = encode_categorial_features_transform(train, categorial_features_encoders)
 columns_to_drop = list(set(categorial_features_columns)& set(train.columns))
train = train.drop(columns_to_drop, axis=1 ).merge(temp, how='left', left_index=True, right_index=True)
 del temp
loaded_model = load_model('./driverdistraction_lr_weights.h5' )<prepare_x_and_y>
temp = encode_categorial_features_transform(test, categorial_features_encoders)
 columns_to_drop = list(set(categorial_features_columns)& set(test.columns))
test = test.drop(columns_to_drop, axis=1 ).merge(temp, how='left', left_index=True, right_index=True)
 del temp
test_data = np.array(testing_data[1001][1] ).reshape(-1,img_size2,img_size1,1 )<predict_on_test>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
preds = model.predict(test_data)
 preds<prepare_output>
y_train = train['isFraud'].copy()
x_train = train.drop('isFraud', axis=1)
 del train
x_test = test.copy()
del test
preds= np.argmax(preds)
 preds<define_variables>
from sklearn.model_selection import KFold, train_test_split
import lightgbm as lgb
from sklearn.metrics import roc_auc_score
import optuna
classes = {0: "safe driving",
1: "texting - right",
2: "talking on the phone - right",
3: "texting - left",
4: "talking on the phone - left",
5: "operating the radio",
6: "drinking",
7: "reaching behind",
8: "hair and makeup",
9: "talking to passenger",
}
for key,value in classes.items() :
if preds==key:
predicted = value
predicted<prepare_x_and_y>
def objective(trial):
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'boosting_type': 'gbdt',
'is_unbalance': False,
'boost_from_average': True,
'num_threads': 4,
'num_leaves': trial.suggest_int('num_leaves', 10, 1000),
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 200),
'min_child_weight': trial.suggest_loguniform('min_child_weight', 0.001, 0.1),
'max_depth': trial.suggest_int('max_depth', 1, 100),
'bagging_fraction' : trial.suggest_loguniform('bagging_fraction',.5,.99),
'feature_fraction' : trial.suggest_loguniform('feature_fraction',.5,.99),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 0.1, 2),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 0.1, 2)
 }
scores = []
cv = KFold(n_splits=5)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train.iloc[train_idx]
y_train_train = y_train.iloc[train_idx]
x_train_valid = x_train.iloc[valid_idx]
y_train_valid = y_train.iloc[valid_idx]
lgb_train = lgb.Dataset(data=x_train_train.astype('float32'), label=y_train_train.astype('float32'))
lgb_valid = lgb.Dataset(data=x_train_valid.astype('float32'), label=y_train_valid.astype('float32'))
lgb_model = lgb.train(params, lgb_train, valid_sets=lgb_valid, verbose_eval=100)
 y = lgb_model.predict(x_train_valid.astype('float32'), num_iteration=lgb_model.best_iteration)
 score = roc_auc_score(y_train_valid.astype('float32'), y)
 print('Fold score:', score)
 scores.append(score)
 average_score = sum(scores)/ len(scores)
 print('Average score:', average_score)
 return average_score
x_test=[]
y_test=[]
for test_id, feature in testing_data:
x_test.append(feature)
 y_test.append(test_id )<define_variables>
params = {
'objective': 'binary',
'metric': 'auc',
'boosting_type': 'gbdt',
'is_unbalance': False,
'boost_from_average': True,
'num_threads': 4,
'num_iterations': 5000,
'learning_rate': 0.006,
'early_stopping_round': 100,
'num_leaves': 396,
'min_data_in_leaf': 39,
'min_child_weight': 0.06046322469906681,
'max_depth': 95,
'bagging_fraction' : 0.7119300452429695,
'feature_fraction' : 0.6121880522551549,
'lambda_l1': 0.30642889371923837,
'lambda_l2': 0.6028242575804484
}
print('features: ', x_test[0])
 print('test_id : ', y_test[0] )<predict_on_test>
n_splits=5
y = np.zeros(x_test.shape[0])
 feature_importances = []
cv = KFold(n_splits=n_splits)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train.iloc[train_idx]
y_train_train = y_train.iloc[train_idx]
x_train_valid = x_train.iloc[valid_idx]
y_train_valid = y_train.iloc[valid_idx]
lgb_train = lgb.Dataset(data=x_train_train.astype('float32'), label=y_train_train.astype('float32'))
lgb_valid = lgb.Dataset(data=x_train_valid.astype('float32'), label=y_train_valid.astype('float32'))
lgb_model = lgb.train(params, lgb_train, valid_sets=lgb_valid, verbose_eval=100, early_stopping_rounds=100)
 y_part = lgb_model.predict(x_test.astype('float32'), num_iteration=lgb_model.best_iteration)
 y += y_part / n_splits
feature_importances.append(lgb_model.feature_importance() )
preds = model.predict(X_test )<save_to_csv>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 submission['isFraud'] = y
submission.to_csv('lightgbm.csv' )