Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<save_to_csv>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os
dataset_submission_lightgbm = pd.DataFrame()
dataset_submission_lightgbm['ID'] = dataset_santandervp_test_ID
dataset_submission_lightgbm['target'] =(( lightgbm_model_predictions_test))
dataset_submission_lightgbm.to_csv('submission_lightgbm.csv', index=False,
float_format='%.2f' )<save_to_csv>
submission1 = pd.read_csv('/kaggle/input/tfidf/submission(27 ).csv')
 submission2 = pd.read_csv('.. /input/tpuinference-super-fast-xlmroberta/submission(47 ).csv' )
dataset_submission_catboost = pd.DataFrame()
dataset_submission_catboost['ID'] = dataset_santandervp_test_ID
dataset_submission_catboost['target'] =(( catboost_model_predictions_test))
dataset_submission_catboost.to_csv('submission_catboost.csv', index=False,
float_format='%.2f' )<import_modules>
submission1['toxic'] = submission1['toxic']*0.1 + submission2['toxic']*0.9
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.externals import joblib
import xgboost as xgb
import lightgbm as lgb 
<define_variables>
submission1.to_csv('submission.csv', index=False )
train_file = '.. /input/train.csv'
test_file = '.. /input/test.csv'<load_from_csv>
sub_path = ".. /input/blending"
all_files = os.listdir(sub_path)
 all_files
train = pd.read_csv(train_file)
 test = pd.read_csv(test_file )<prepare_x_and_y>
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "jigsaw" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]
test_ID = test['ID']
y_train = train['target']
y_train = np.log1p(y_train )<drop_column>
concat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )
train.drop("ID", axis = 1, inplace = True)
 train.drop("target", axis = 1, inplace = True)
 test.drop("ID", axis = 1, inplace = True )<normalization>
cutoff_lo = 0.7
cutoff_hi = 0.3
NUM_OF_DECIMALS = 32
train = train.round(NUM_OF_DECIMALS)
 test = test.round(NUM_OF_DECIMALS )<define_variables>
concat_sub['toxic'] = concat_sub['jigsaw_mean']
concat_sub[['toxic']].to_csv('submission2.csv',
index=False, float_format='%.6f' )
train_zeros = pd.DataFrame({'Percent_zero':(( train.values)==0 ).mean(axis=0),
'Column' : train.columns})
 high_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] < 0.70].values
low_vol_columns = train_zeros['Column'][train_zeros['Percent_zero'] >= 0.70].values 
<categorify>
concat_sub['toxic'] = concat_sub['jigsaw_median']
concat_sub[['toxic']].to_csv('submission1.csv',
index=False, float_format='%.6f' )
train = train.replace({0:np.nan})
 test = test.replace({0:np.nan} )<feature_engineering>
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission3.csv',
index=False, float_format='%.6f' )
cluster_sets = {"low":low_vol_columns, "high":high_vol_columns}
for cluster_key in cluster_sets:
for df in [train,test]:
df["count_not0_"+cluster_key] = df[cluster_sets[cluster_key]].count(axis=1)
 df["sum_"+cluster_key] = df[cluster_sets[cluster_key]].sum(axis=1)
 df["var_"+cluster_key] = df[cluster_sets[cluster_key]].var(axis=1)
 df["median_"+cluster_key] = df[cluster_sets[cluster_key]].median(axis=1)
 df["mean_"+cluster_key] = df[cluster_sets[cluster_key]].mean(axis=1)
 df["std_"+cluster_key] = df[cluster_sets[cluster_key]].std(axis=1)
 df["max_"+cluster_key] = df[cluster_sets[cluster_key]].max(axis=1)
 df["min_"+cluster_key] = df[cluster_sets[cluster_key]].min(axis=1)
 df["skew_"+cluster_key] = df[cluster_sets[cluster_key]].skew(axis=1)
 df["kurtosis_"+cluster_key] = df[cluster_sets[cluster_key]].kurtosis(axis=1)
<drop_column>
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_mean']))
concat_sub[['toxic']].to_csv('submission4.csv',
index=False, float_format='%.6f' )
<train_model><EOS> 
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission5.csv',
index=False, float_format='%.6f' )