Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
sns.set()
pd.DataFrame(result_dict_lgb['oof'] ).to_csv('lgb_oof.csv', index=False )<set_options>
from sklearn.preprocessing import OneHotEncoder, QuantileTransformer
from sklearn.model_selection import KFold
from sklearn.compose import ColumnTransformer
from sklearn.decomposition import PCA
from sklearn.pipeline import make_pipeline
from sklearn.multioutput import MultiOutputClassifier
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
 print(os.listdir(".. /input/ieee-fraud-detection"))
print("Ready!" )<load_from_csv>
from xgboost import XGBClassifier
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras import callbacks
import keras
from sklearn.metrics import log_loss
from keras.regularizers import L1, L2
%%time
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
 print("Data is loaded!" )<count_missing_values>
drug = pd.read_csv('.. /input/lish-moa/train_features.csv')
 target = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 drug.set_index('sig_id', inplace= True)
 target.set_index('sig_id', inplace= True)
 treat_drug = drug.query('cp_type == "trt_cp"')
 treat_target = target.loc[treat_drug.index]
missing_values_count = train_transaction.isnull().sum()
print(missing_values_count[0:10])
 total_cells = np.product(train_transaction.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )<load_from_csv>
cat_cols = drug.select_dtypes(include = 'O' ).columns.tolist()
num_cols = drug.select_dtypes(exclude = 'O' ).columns.tolist()
gene_features = [i for i in num_cols if i.startswith('g-')]
cell_viability = [i for i in num_cols if i.startswith('c-')]
cat_cols2 = cat_cols + ['cp_time']
num_cols2 = num_cols
num_cols.remove('cp_time' )
%%time
folder_path = '.. /input/ieee-fraud-detection/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
' )<feature_engineering>
qt = QuantileTransformer()
def data_preprocessing(dataframe, only_treatment = True, fit = False, transform = False):
df = dataframe.copy()
if fit:
df[num_cols] = qt.fit_transform(df[num_cols])
 if transform:
df[num_cols] = qt.transform(df[num_cols])
 df["cp_dose"] = df.cp_dose.map({"D1": 0, "D2":1})
 df["cp_time"] = df.cp_time.map({24: 0,48: 1, 72: 2})
 if only_treatment:
df = df.drop("cp_type", 1)
 else:
df["cp_type"] = df.cp_type.map({"trt_cp": 1, "ctl_vehicle":0})
 return df
drug_cleaned = data_preprocessing(dataframe= drug, only_treatment= False, fit= True, transform= False)
 drug_treatment = data_preprocessing(dataframe= drug, only_treatment= True,fit= True, transform= False )
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe<split>
def for_bayes_optimization2(dimension):
[dl1,dl2,dl3,dl4,dp1,dp2,dp3,dp4,regu,regu_val,activation,learning_rate] = dimension
if(regu == 'l2'):
act_reg = keras.regularizers.l2(regu_val)
 if(regu =='l1'):
act_reg = keras.regularizers.l1(regu_val)
 lr = callbacks.ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 5, verbose = 0)
 es = callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, mode = 'min', baseline = 0.3 ,
restore_best_weights=False, patience= 30, verbose = 0)
 adam = keras.optimizers.Adam(learning_rate = learning_rate)
 model = Sequential()
model.add(Dense(dl1, input_dim = x_train.shape[1], activation = activation, activity_regularizer = act_reg))
model.add(Dropout(dp1))
model.add(Dense(dl2, activation = activation))
model.add(Dropout(dp2))
model.add(Dense(dl3, activation = activation))
model.add(Dropout(dp3))
model.add(Dense(dl4, activation = activation))
model.add(Dropout(dp4))
model.add(Dense(y_train.shape[1], activation = 'sigmoid'))
model.compile(optimizer = adam, loss = 'binary_crossentropy', metrics = ['AUC'])
 model.fit(x = x_train, y = y_train, validation_data =(x_val, y_val), epochs = 200, batch_size = 128, callbacks = [es], verbose = 0)
 log_loss_data = log_loss(np.ravel(y_val), np.ravel(model.predict_proba(x_val)) , eps = 1e-7)
 return model
train_identity = id_split(train_identity)
 test_identity = id_split(test_identity )<merge>
best_set_from_baysian_optimization = [2048, 1982, 708, 470, 0.6067766671093088, 0.1, 0.4973213653064633, 0.5950996340056243, 'l1', 1e-05, 'swish', 0.0001]
print('Merging data...')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()<define_variables>
submission_test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission_test_prob = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_test_cleaned = data_preprocessing(dataframe= submission_test, only_treatment= False, fit= False, transform= True)
 submission_test_prob.set_index('sig_id', inplace= True)
 submission_test_cleaned.set_index('sig_id', inplace = True)
 submission_test_cleaned
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']<drop_column>
submission_test_prob[:] = np.zeros(submission_test_prob.shape)
 submission_test_prob
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT' )<drop_column>
n_splits = 5
sub_file = submission_test_cleaned
sub_file_all_predict = np.zeros(submission_test_prob.shape)
 nn_loss = []
xgb_loss = []
combined_loss = []
for seed in [10, 20, 30]:
for e,(train, val)in enumerate(KFold(n_splits = n_splits, shuffle = True, random_state = seed ).split(drug_cleaned, target)) :
x_train, y_train = drug_cleaned.iloc[train], target.iloc[train]
x_val, y_val = drug_cleaned.iloc[val], target.iloc[val]
model = for_bayes_optimization2(best_set_from_baysian_optimization)
 nn_predict = model.predict_proba(x_val)
 sub_file_nn_predict = model.predict_proba(sub_file)
 nn_loss_temp = log_loss(np.ravel(y_val), np.ravel(nn_predict), eps = 1e-7)
 nn_loss.append(nn_loss_temp)
 print(f"NN_log_loss fold {e}, seed {seed}: ", nn_loss_temp)
 xgb = MultiOutputClassifier(XGBClassifier(tree_method = 'gpu_hist', n_estimators = 130, max_depth = 3, reg_alpha = 2, min_child_weight = 2,
gamma = 3, learning_rate = 0.0580666601841646, colsample_bytree = 0.58))
xgb.fit(x_train, y_train)
 xgb_predict = np.array(xgb.predict_proba(x_val)) [:,:,1].T
xgb_loss_temp = log_loss(np.ravel(y_val), np.ravel(xgb_predict), eps = 1e-7)
 xgb_loss.append(xgb_loss_temp)
 sub_file_xgb_predict = np.array(xgb.predict_proba(sub_file)) [:,:,1].T
avg_sub_file_predict =(sub_file_nn_predict + sub_file_xgb_predict)/2
sub_file_all_predict = sub_file_all_predict + avg_sub_file_predict
combined_loss_temp = log_loss(np.ravel(y_val), np.ravel(( nn_predict + xgb_predict)/2), eps = 1e-7)
 combined_loss.append(combined_loss_temp)
 print(f"xgb_log_loss fold {e}, seed {seed}: ", xgb_loss_temp)
 print(f"combined_loss fold {e}, seed {seed}: ", combined_loss_temp)
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<categorify>
print("Average log loss of NN is :", np.mean(nn_loss), " and standard deviation: ", np.std(nn_loss))
print("Average log loss of Xgboost is :", np.mean(xgb_loss), " and standard deviation: ", np.std(xgb_loss))
print("Combined log loss is :", np.mean(combined_loss), " and standard deviation: ", np.std(combined_loss))
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )<feature_engineering>
final_predictions = sub_file_all_predict/(n_splits * 3 )
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))<feature_engineering>
submission_test_prob[:] = final_predictions
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<data_type_conversions>
submission_test_prob = np.clip(submission_test_prob, 0.0005, 0.99)
 submission_test_prob.iloc[submission_test.query('cp_type == "ctl_vehicle"' ).index] = 0.0
<drop_column><EOS> 
submission_test_prob.to_csv('submission.csv' )