Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<import_modules>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import gc
import xgboost as xgb
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split<load_from_csv>
%matplotlib inline
def bureau_balance_preprocess() :
bureau = pd.read_csv(".. /input/bureau.csv")
 balance = pd.read_csv(".. /input/bureau_balance.csv")
 cat_cols = []
for i in balance.columns.values:
if balance[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in balance.columns if col not in cat_cols][1:]
balance = pd.get_dummies(balance, columns = ['STATUS'])
 cat_cols = [c for c in balance.columns.tolist() if c not in [cat_cols, num_cols]][2:]
cat_aggs = {}
num_aggs = {}
for i in cat_cols:
cat_aggs[i] = ['mean']
for j in num_cols:
num_aggs[j] = ['size']
balance = balance.groupby('SK_ID_BUREAU' ).agg({**num_aggs, **cat_aggs})
 balance.columns = [i[0] + "_" + i[1]for i in balance.columns.tolist() ]
balance.reset_index(inplace = True)
 cat_cols = []
for i in bureau.columns.values:
if bureau[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in bureau.columns if col not in cat_cols]
bureau = pd.get_dummies(bureau, columns = cat_cols)
 df = bureau.merge(balance, how='left', on='SK_ID_BUREAU')
 df.drop(['SK_ID_BUREAU'], axis = 1, inplace = True)
 aggregations = {}
for i in df.columns.tolist() [1:]:
aggregations[i] = ['min', 'max', 'mean', 'median', 'var', 'size']
df = df.groupby('SK_ID_CURR' ).agg(aggregations)
 df.columns = [i[0] + "_" + i[1]for i in df.columns.tolist() ]
df = df.reset_index()
return df<load_from_csv>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
def prev_application() :
df = pd.read_csv(".. /input/previous_application.csv")
 df.loc[df['AMT_CREDIT'] > 6000000, 'AMT_CREDIT'] = np.nan
df.loc[df['SELLERPLACE_AREA'] > 3500000, 'SELLERPLACE_AREA'] = np.nan
df['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 df['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 df['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 df['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 df['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 df['prev missing'] = df.isnull().sum(axis = 1 ).values
df['prev AMT_APPLICATION / AMT_CREDIT'] = df['AMT_APPLICATION'] / df['AMT_CREDIT']
df['prev AMT_APPLICATION - AMT_CREDIT'] = df['AMT_APPLICATION'] - df['AMT_CREDIT']
df['prev AMT_APPLICATION - AMT_GOODS_PRICE'] = df['AMT_APPLICATION'] - df['AMT_GOODS_PRICE']
df['prev AMT_GOODS_PRICE - AMT_CREDIT'] = df['AMT_GOODS_PRICE'] - df['AMT_CREDIT']
df['prev DAYS_FIRST_DRAWING - DAYS_FIRST_DUE'] = df['DAYS_FIRST_DRAWING'] - df['DAYS_FIRST_DUE']
df['prev DAYS_TERMINATION less -500'] =(df['DAYS_TERMINATION'] < -500 ).astype(int)
 df.rename(columns = lambda x : 'PREV_' + x if x not in ['SK_ID_PREV', 'SK_ID_CURR'] else x, inplace = True)
 cat_cols = []
for i in df.columns.values:
if df[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in df.columns[2:] if col not in cat_cols]
df = pd.get_dummies(df, columns = cat_cols)
 new_cat_cols = [c for c in df.columns.tolist() if c not in [cat_cols, num_cols]][2:]
cat_aggs = {}
num_aggs = {}
for i in new_cat_cols:
cat_aggs[i] = ['mean']
for j in num_cols:
num_aggs[j] = ['min', 'max', 'mean', 'median', 'var', 'size']
df.drop(['SK_ID_PREV'], axis = 1, inplace = True)
 df = df.groupby('SK_ID_CURR' ).agg({**num_aggs, **cat_aggs})
 df.columns = [i[0] + "_" + i[1]for i in df.columns.tolist() ]
return df.reset_index()<load_from_csv>
import transformers
from tokenizers import BertWordPieceTokenizer
from tqdm.notebook import tqdm
from kaggle_datasets import KaggleDatasets
def posh_cash_balance() :
df = pd.read_csv(".. /input/POS_CASH_balance.csv")
 df.loc[df['CNT_INSTALMENT_FUTURE'] > 60, 'CNT_INSTALMENT_FUTURE'] = np.nan
df.drop(['SK_ID_PREV'], axis = 1, inplace = True)
 df['pos CNT_INSTALMENT more CNT_INSTALMENT_FUTURE'] = \ 
(df['CNT_INSTALMENT'] > df['CNT_INSTALMENT_FUTURE'] ).astype(int)
 cat_cols = []
for i in df.columns.values:
if df[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [c for c in df.columns.tolist() if c not in cat_cols][1:]
df = pd.get_dummies(df, columns = ['NAME_CONTRACT_STATUS'])
 cat_cols = [c for c in df.columns.tolist() if c not in [cat_cols, num_cols]][1:]
cat_aggs = {}
num_aggs = {}
for i in cat_cols:
cat_aggs[i] = ['mean']
for j in num_cols:
num_aggs[j] = ['min', 'max', 'mean', 'median', 'var', 'size']
df = df.groupby('SK_ID_CURR' ).agg({**num_aggs, **cat_aggs})
 df.columns = [i[0] + "_" + i[1]for i in df.columns.tolist() ]
df.reset_index(inplace = True)
 df['CNT_INSTALMENT_RATIO'] = df['CNT_INSTALMENT_mean'] / df['MONTHS_BALANCE_mean']
df['CNT_INSTALMENT_FUTURE_RATIO'] = df['CNT_INSTALMENT_FUTURE_mean'] / df['MONTHS_BALANCE_mean']
df['CNT_INSTALMENT_SUM_RATIO'] =(df['CNT_INSTALMENT_mean'] + df['CNT_INSTALMENT_FUTURE_mean'])/ df['MONTHS_BALANCE_mean']
return df<feature_engineering>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
def cc_balance() :
df = pd.read_csv(".. /input/credit_card_balance.csv")
 df.loc[df['AMT_PAYMENT_CURRENT'] > 4000000, 'AMT_PAYMENT_CURRENT'] = np.nan
df.loc[df['AMT_CREDIT_LIMIT_ACTUAL'] > 1000000, 'AMT_CREDIT_LIMIT_ACTUAL'] = np.nan
df['card missing'] = df.isnull().sum(axis = 1 ).values
df['card SK_DPD - MONTHS_BALANCE'] = df['SK_DPD'] - df['MONTHS_BALANCE']
df['card SK_DPD_DEF - MONTHS_BALANCE'] = df['SK_DPD_DEF'] - df['MONTHS_BALANCE']
df['card SK_DPD - SK_DPD_DEF'] = df['SK_DPD'] - df['SK_DPD_DEF']
df['card AMT_TOTAL_RECEIVABLE - AMT_RECIVABLE'] = df['AMT_TOTAL_RECEIVABLE'] - df['AMT_RECIVABLE']
df['card AMT_TOTAL_RECEIVABLE - AMT_RECEIVABLE_PRINCIPAL'] = df['AMT_TOTAL_RECEIVABLE'] - df['AMT_RECEIVABLE_PRINCIPAL']
df['card AMT_RECIVABLE - AMT_RECEIVABLE_PRINCIPAL'] = df['AMT_RECIVABLE'] - df['AMT_RECEIVABLE_PRINCIPAL']
df['card AMT_BALANCE - AMT_RECIVABLE'] = df['AMT_BALANCE'] - df['AMT_RECIVABLE']
df['card AMT_BALANCE - AMT_RECEIVABLE_PRINCIPAL'] = df['AMT_BALANCE'] - df['AMT_RECEIVABLE_PRINCIPAL']
df['card AMT_BALANCE - AMT_TOTAL_RECEIVABLE'] = df['AMT_BALANCE'] - df['AMT_TOTAL_RECEIVABLE']
df['card AMT_DRAWINGS_CURRENT - AMT_DRAWINGS_ATM_CURRENT'] = df['AMT_DRAWINGS_CURRENT'] - df['AMT_DRAWINGS_ATM_CURRENT']
df['card AMT_DRAWINGS_CURRENT - AMT_DRAWINGS_OTHER_CURRENT'] = df['AMT_DRAWINGS_CURRENT'] - df['AMT_DRAWINGS_OTHER_CURRENT']
df['card AMT_DRAWINGS_CURRENT - AMT_DRAWINGS_POS_CURRENT'] = df['AMT_DRAWINGS_CURRENT'] - df['AMT_DRAWINGS_POS_CURRENT']
df.rename(columns = lambda x : 'PREV_' + x if x not in ['SK_ID_PREV', 'SK_ID_CURR'] else x, inplace = True)
 cat_cols = []
for i in df.columns.values:
if df[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in df.columns[2:] if col not in cat_cols]
df = pd.get_dummies(df, columns = cat_cols)
 new_cat_cols = [c for c in df.columns.tolist() if c not in [cat_cols, num_cols]][2:]
cat_aggs = {}
num_aggs = {}
for i in new_cat_cols:
cat_aggs[i] = ['mean']
for j in num_cols:
num_aggs[j] = ['min', 'max', 'mean', 'median', 'var', 'size']
df.drop(['SK_ID_PREV'], axis = 1, inplace = True)
 df = df.groupby('SK_ID_CURR' ).agg({**num_aggs, **cat_aggs})
 df.columns = [i[0] + "_" + i[1]for i in df.columns.tolist() ]
return df.reset_index()<load_from_csv>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
dense_layer = Dense(224, activation='relu' )(cls_token)
 out = Dense(1, activation='sigmoid' )(dense_layer)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def installments() :
df = pd.read_csv(".. /input/installments_payments.csv")
 df.loc[df['NUM_INSTALMENT_VERSION'] > 70, 'NUM_INSTALMENT_VERSION'] = np.nan
df.loc[df['DAYS_ENTRY_PAYMENT'] < -4000, 'DAYS_ENTRY_PAYMENT'] = np.nan
df['ins DAYS_ENTRY_PAYMENT - DAYS_INSTALMENT'] = df['DAYS_ENTRY_PAYMENT'] - df['DAYS_INSTALMENT']
df['ins NUM_INSTALMENT_NUMBER_100'] =(df['NUM_INSTALMENT_NUMBER'] == 100 ).astype(int)
 df['ins DAYS_INSTALMENT more NUM_INSTALMENT_NUMBER'] =(df['DAYS_INSTALMENT'] > df['NUM_INSTALMENT_NUMBER'] * 50 / 3 - 11500 / 3 ).astype(int)
 df['ins AMT_INSTALMENT - AMT_PAYMENT'] = df['AMT_INSTALMENT'] - df['AMT_PAYMENT']
df['ins AMT_PAYMENT / AMT_INSTALMENT'] = df['AMT_PAYMENT'] / df['AMT_INSTALMENT']
df.drop([])
 df.rename(columns = lambda x : 'PREV_' + x if x not in ['SK_ID_PREV', 'SK_ID_CURR'] else x, inplace = True)
 cat_cols = []
for i in df.columns.values:
if df[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in df.columns[2:] if col not in cat_cols]
df = pd.get_dummies(df, columns = cat_cols)
 new_cat_cols = [c for c in df.columns.tolist() if c not in [cat_cols, num_cols]][2:]
cat_aggs = {}
num_aggs = {}
for i in new_cat_cols:
cat_aggs[i] = ['mean']
for j in num_cols:
num_aggs[j] = ['min', 'max', 'mean', 'median', 'var', 'size']
df.drop(['SK_ID_PREV'], axis = 1, inplace = True)
 df = df.groupby('SK_ID_CURR' ).agg({**num_aggs, **cat_aggs})
 df.columns = [i[0] + "_" + i[1]for i in df.columns.tolist() ]
return df.reset_index()<load_from_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def get_beautiful_data() :
print('Getting train data...')
 train = pd.read_csv(".. /input/application_train.csv")
 print('Getting test data...')
 test = pd.read_csv(".. /input/application_test.csv")
 y = train.pop('TARGET')
 index_end = y.shape[0]
test_index = test.pop('SK_ID_CURR')
 cat_cols = []
for i in train.columns.values:
if train[i].dtype == 'object':
cat_cols.append(i)
 num_cols = [col for col in train.columns if col not in cat_cols]
print('Merging train and tests data...')
 df = pd.concat([train, test], axis = 0, ignore_index = True)
 del train, test
gc.collect()
print('Merge complete.')
 print('Fetching bureau and balance data...')
 bureau = bureau_balance_preprocess()
print('Merging to main data frame...')
 df = df.merge(bureau, how = 'left', on = 'SK_ID_CURR')
 del bureau
gc.collect()
print('Merge complete.')
 print('Fetching previous_application data...')
 prev = prev_application()
print('Merging to main data frame...')
 df = df.merge(prev, how = 'left', on = 'SK_ID_CURR')
 del prev
gc.collect()
print('Merge complete.')
 print('Fetching POS_CASH_balance data...')
 pos = posh_cash_balance()
print('Merging to main data frame...')
 df = df.merge(pos, how = 'left', on = 'SK_ID_CURR')
 del pos
gc.collect()
print('Merge complete.')
 print('Fetching credit_card_balance data...')
 cc = cc_balance()
print('Merging to main data frame...')
 df = df.merge(cc, how = 'left', on = 'SK_ID_CURR')
 del cc
gc.collect()
print('Merge complete.')
 print('Fetching installments_payments data...')
 inst = installments()
print('Merging to main data frame...')
 df = df.merge(inst, how = 'left', on = 'SK_ID_CURR')
 del inst
gc.collect()
print('Merge complete')
 docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]
live = [_f for _f in df.columns if('FLAG_' in _f)&('FLAG_DOC' not in _f)&('_FLAG_' not in _f)]
print('Adding more features...')
 df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)
 df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)
 df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] /(1 + df['AMT_INCOME_TOTAL'])
 df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())
 df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
print('Done.')
 df = pd.get_dummies(df, columns = cat_cols, dummy_na= True)
 X = df.iloc[:index_end, 1:]
X_test = df.iloc[index_end:, 1:]
scale_pos_weight = y.value_counts() [0]/ y.value_counts() [1]
X, X_test = X.align(X_test, join = 'inner', axis = 1)
 return X, X_test, y, scale_pos_weight, test_index<split>
AUTO = tf.data.experimental.AUTOTUNE
X, X_test, y, scale_pos_weight, test_index = get_beautiful_data()<data_type_conversions>
EPOCHS = 3
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
MAX_LEN = 192
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper.head()<define_variables>
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=True)
 fast_tokenizer
to_drop = [column for column in upper.columns if any(upper[column] > threshold)]
print('There are %d columns to remove.' %(len(to_drop)) )<train_model>
DATA_PATH = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"
X = X.drop(columns = to_drop)
 X_test = X_test.drop(columns = to_drop)
 print('Training shape: ', X.shape)
 print('Testing shape: ', X_test.shape )<sort_values>
train1 = pd.read_csv(os.path.join(DATA_PATH, "jigsaw-toxic-comment-train.csv"))
train2 = pd.read_csv(os.path.join(DATA_PATH, "jigsaw-unintended-bias-train.csv"))
train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv(os.path.join(DATA_PATH, 'validation.csv'))
test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))
sub = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))
train_missing =(X.isnull().sum() / len(X)).sort_values(ascending = False)
 train_missing.head()<sort_values>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=3982)
 ])
test_missing =(X_test.isnull().sum() / len(X_test)).sort_values(ascending = False)
 test_missing.head()<filter>
x_train = fast_encode(train.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
train_missing = train_missing.index[train_missing > 0.75]
test_missing = test_missing.index[test_missing > 0.75]
all_missing = list(set(set(train_missing)| set(test_missing)))
 print('There are %d columns with more than 75%% missing values' % len(all_missing))<train_model>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
X = X.drop(columns = all_missing)
 X_test = X_test.drop(columns = all_missing)
 print('Training shape: ', X.shape)
 print('Testing shape: ', X_test.shape )<choose_model_class>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
clf = xgb.XGBClassifier(learning_rate = 0.025,
n_estimators = 10000,
max_depth = 8,
min_child_weight = 1,
subsample = 0.8,
colsample_bytree = 0.7,
colsample_bylevel = 0.7,
objective = 'binary:logistic',
n_jobs = -1,
scale_pos_weight = scale_pos_weight,
silent = True )<split>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state = 42 )<train_model>
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )
clf.fit(X_train, y_train, eval_set = [(X_train, y_train),(X_val, y_val)], eval_metric= 'auc', verbose= 100, early_stopping_rounds= 400 )<save_to_csv>
submission1 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-1.csv'))
submission2 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-2.csv'))
submission3 = pd.read_csv(os.path.join(EXTERNAL_DATA_PATH, 'submission-3.csv'))
result = clf.predict_proba(X_test)
 submit = pd.DataFrame({'SK_ID_CURR': test_index, 'TARGET': result[:, 1]})
 submit.to_csv('solution.csv', index = False )<set_options>
submission1['toxic'] = submission1['toxic']*0.05 + submission2['toxic']*0.15 + submission3['toxic']*0.8
submission1.to_csv('submission-1.csv', index=False )