Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<count_unique_values>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os
print(f'Total unique patients are {train_df.Patient.nunique() } out of total {len(train_df.Patient)} patients' )<count_values>
submission1 = pd.read_csv('/kaggle/input/tfidf/submission(27 ).csv')
 submission2 = pd.read_csv('.. /input/tpuinference-super-fast-xlmroberta/submission(47 ).csv' )
train_df.groupby('Sex')['SmokingStatus'].value_counts()<define_variables>
s = []
s1 = submission1['toxic'].tolist()
s2 = submission2['toxic'].tolist()
for i in range(len(s1)) :
s.append(( s1[i]+s2[i])/2)
 submission1['toxic'] = s
source_path = Path('.. /input/osic-pulmonary-fibrosis-progression')
 source_files = os.listdir(source_path)
 print(source_files )<define_variables>
submission1.to_csv('submission.csv', index=False )
train_path = source_path/'train'
train_files = get_dicom_files(train_path)
 train_files<load_pretrained>
sub_path = ".. /input/blending"
all_files = os.listdir(sub_path)
 all_files
dicom_img = dcmread(train_files[0])
 dicom_img<count_duplicates>
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "jigsaw" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]
train_df[train_df.duplicated(subset = ['Patient','Weeks'])]<remove_duplicates>
concat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )
train_df.drop_duplicates(keep=False, inplace = True, subset=['Patient','Weeks'] )<load_from_csv>
cutoff_lo = 0.7
cutoff_hi = 0.3
submission_df = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<rename_columns>
concat_sub['toxic'] = concat_sub['jigsaw_mean']
concat_sub[['toxic']].to_csv('submission2.csv',
index=False, float_format='%.6f' )
temp_sub_df = submission_df['Patient_Week'].str.split('_', expand = True)
 temp_sub_df.rename(columns = {0: 'Patient', 1: 'Weeks'}, inplace = True )<concatenate>
concat_sub['toxic'] = concat_sub['jigsaw_median']
concat_sub[['toxic']].to_csv('submission1.csv',
index=False, float_format='%.6f' )
submission_df = pd.concat([submission_df, temp_sub_df], axis = 1)
 submission_df = submission_df[['Patient','Weeks','Confidence','Patient_Week']]<merge>
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission3.csv',
index=False, float_format='%.6f' )
submission_df = submission_df.merge(test_df.drop('Weeks', axis = 1), on = 'Patient' )<concatenate>
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_mean']))
concat_sub[['toxic']].to_csv('submission4.csv',
index=False, float_format='%.6f' )
train_df['data_type'] = 'Train'
test_df['data_type'] = 'Val'
submission_df['data_type'] = 'Test'
combined_df = train_df.append([test_df, submission_df] )<feature_engineering>
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission5.csv',
index=False, float_format='%.6f' )