Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<categorify>
!pip install -q googletrans
translator = Translator()
GRU,Embedding, Flatten, Dropout, Add, concatenate, MaxPooling1D,
GlobalAveragePooling1D, GlobalMaxPooling1D,
GlobalMaxPool1D,SpatialDropout1D)
 optimizers, layers, callbacks)
 sns.set(style="darkgrid" )
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test)[:,0]*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )<import_modules>
dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'
train_set1 = pd.read_csv(os.path.join(dir, 'jigsaw-toxic-comment-train.csv'))
train_set2 = pd.read_csv(os.path.join(dir, 'jigsaw-unintended-bias-train.csv'))
train_set2.toxic = train_set2.toxic.round().astype(int)
 valid = pd.read_csv(os.path.join(dir, 'validation.csv'))
test = pd.read_csv(os.path.join(dir, 'test.csv'))
from sklearn import *
import sklearn
import pandas as pd
import numpy as np
import lightgbm as lgb<load_from_csv>
train = pd.concat([
train_set1[['comment_text', 'toxic']],
train_set2[['comment_text', 'toxic']].query('toxic==1'),
train_set2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )
train = pd.read_csv('.. /input/training_variants')
 test = pd.read_csv('.. /input/stage2_test_variants.csv')
 trainx = pd.read_csv('.. /input/training_text', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"])
 testx = pd.read_csv('.. /input/stage2_test_text.csv', sep="\|\|", engine='python', header=None, skiprows=1, names=["ID","Text"] )<merge>
print(train.toxic.value_counts())
 sns.countplot(train.toxic )
train = pd.merge(train, trainx, how='left', on='ID' ).fillna('')
 train = train.drop(["ID"], axis=1)
 test = pd.merge(test, testx, how='left', on='ID' ).fillna('')
 pid = test['ID'].values<load_from_csv>
print(valid.toxic.value_counts())
 sns.countplot(valid.toxic )
df_variants_test = pd.read_csv('.. /input/test_variants', usecols=['ID', 'Gene', 'Variation'])
 df_text_test = pd.read_csv('.. /input/test_text', sep='\|\|', engine='python',
skiprows=1, names=['ID', 'Text'])
 df_variants_test['Text'] = df_text_test['Text']
df_test = df_variants_test
df_labels_test = pd.read_csv('.. /input/stage1_solution_filtered.csv')
 df_labels_test['Class'] = pd.to_numeric(df_labels_test.drop('ID', axis=1 ).idxmax(axis=1 ).str[5:])
 df_test = df_test.merge(df_labels_test[['ID', 'Class']], on='ID', how='left' ).drop('ID', axis=1)
 df_test = df_test[df_test['Class'].notnull() ]
df_stage_2_train = pd.concat([train, df_test])
 df_stage_2_train.reset_index(drop=True, inplace=True)
 train = df_stage_2_train
df_stage_2_train.info()<prepare_x_and_y>
print(valid.lang.value_counts())
 sns.countplot(valid.lang )
y = train['Class'].values
train = train.drop(['Class'], axis=1)
 train.info()<feature_engineering>
print(test.lang.value_counts())
 sns.countplot(test.lang )
df_all = pd.concat(( train, test), axis=0, ignore_index=True)
 df_all['Gene_Share'] = df_all.apply(lambda r: sum([1 for w in r['Gene'].split(' ')if w in r['Text'].split(' ')]), axis=1)
 df_all['Variation_Share'] = df_all.apply(lambda r: sum([1 for w in r['Variation'].split(' ')if w in r['Text'].split(' ')]), axis=1 )<categorify>
def fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in range(0, len(texts), chunk_size):
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
for i in range(56):
df_all['Gene_'+str(i)] = df_all['Gene'].map(lambda x: str(x[i])if len(x)>i else '')
 df_all['Variation'+str(i)] = df_all['Variation'].map(lambda x: str(x[i])if len(x)>i else '' )<count_unique_values>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
gen_var_lst = sorted(list(train.Gene.unique())+ list(train.Variation.unique()))
print(len(gen_var_lst))
gen_var_lst = [x for x in gen_var_lst if len(x.split(' ')) ==1]
print(len(gen_var_lst))
i_ = 0<count_values>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 EPOCHS = 4
BATCH_SIZE = 16* strategy.num_replicas_in_sync
MODEL = 'jplu/tf-xlm-roberta-large'
MAX_LEN = 224
for gen_var_lst_itm in gen_var_lst:
if i_ % 100 == 0: print(i_)
 df_all['GV_'+str(gen_var_lst_itm)] = df_all['Text'].map(lambda x: str(x ).count(str(gen_var_lst_itm)))
 i_ += 1<categorify>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
for c in df_all.columns:
if df_all[c].dtype == 'object':
if c in ['Gene','Variation']:
lbl = preprocessing.LabelEncoder()
df_all[c+'_lbl_enc'] = lbl.fit_transform(df_all[c].values)
 df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
 df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x ).split(' ')))
 elif c != 'Text':
lbl = preprocessing.LabelEncoder()
df_all[c] = lbl.fit_transform(df_all[c].values)
 if c=='Text':
df_all[c+'_len'] = df_all[c].map(lambda x: len(str(x)))
 df_all[c+'_words'] = df_all[c].map(lambda x: len(str(x ).split(' ')) )<categorify>
%%time
x_train = regular_encode(train.comment_text.values,
tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values,
tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer,
maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
train = df_all.iloc[:len(train)]
test = df_all.iloc[len(train):]
class cust_regression_vals(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
def fit(self, x, y=None):
return self
def transform(self, x):
x = x.drop(['Gene', 'Variation','ID','Text'],axis=1 ).values
return x
class cust_txt_col(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):
def __init__(self, key):
self.key = key
def fit(self, x, y=None):
return self
def transform(self, x):
return x[self.key].apply(str )<feature_engineering>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
print('Pipeline...')
 fp = pipeline.Pipeline([ 
('union', pipeline.FeatureUnion(
 n_jobs = -1,
transformer_list = [ 
('standard', cust_regression_vals()), 
('pi1', pipeline.Pipeline([('Gene', cust_txt_col('Gene')) ,('count_Gene', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 10))),('tsvd1', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12)) ])) , 
('pi2', pipeline.Pipeline([('Variation', cust_txt_col('Variation')) ,('count_Variation', feature_extraction.text.CountVectorizer(analyzer=u'char', ngram_range=(1, 10))),('tsvd2', decomposition.TruncatedSVD(n_components=20, n_iter=25, random_state=12)) ])) , 
('pi3', pipeline.Pipeline([('Text', cust_txt_col('Text')) ,('tfidf_Text', feature_extraction.text.TfidfVectorizer(ngram_range=(1, 2))),('tsvd3', decomposition.TruncatedSVD(n_components=50, n_iter=25, random_state=12)) ]))
])
)])
 train = fp.fit_transform(train); print(train.shape)
 test = fp.transform(test); print(test.shape)
 y = y - 1<train_model>
def label_smoothing(y_true,y_pred):
return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.15)
 def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed
denom = 0
for i in range(2):
params = {
'learning_rate': 0.01,
'max_depth': 6,
'num_leaves': 50,
'objective': 'multiclass',
'metric': 'multi_logloss',
'num_class': 9
}
x1, x2, y1, y2 = model_selection.train_test_split(train, y, test_size=0.15, random_state=i)
 train_set = lgb.Dataset(x1, y1)
 valid_set = lgb.Dataset(x2, y2)
 model = lgb.train(params, train_set, 1000, [valid_set], ["val"], verbose_eval=50, early_stopping_rounds=50)
 score1 = metrics.log_loss(y2, model.predict(x2), labels = list(range(9)))
 print(score1)
 if denom != 0:
pred = model.predict(test)
 preds += pred
else:
pred = model.predict(test)
 preds = pred.copy()
denom += 1
submission = pd.DataFrame(pred, columns=['class'+str(c+1)for c in range(9)])
 submission['ID'] = pid
submission.to_csv('submission_xgb_fold_' + str(i)+ '.csv', index=False )<save_to_csv>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Dropout(0.3 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss=label_smoothing,
metrics=[tf.keras.metrics.AUC() ])
 return model
preds /= denom
submission = pd.DataFrame(preds, columns=['class'+str(c+1)for c in range(9)])
 submission['ID'] = pid
submission.to_csv('submission_xgb.csv', index=False)
 submission.head()<import_modules>
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model, ensemble
from sklearn.metrics import mean_squared_error, mean_absolute_error
import tensorflow as tf
from tqdm.notebook import tqdm
import os
from PIL import Image<load_from_csv>
calls = callback()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
callbacks = calls,
epochs=EPOCHS
)
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<remove_duplicates>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<merge><EOS> 
sub = pd.read_csv(dir + '/sample_submission.csv')
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )