Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<categorify>
import sys, os, re, csv, codecs, numpy as np, pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
X_test = sc.transform(test.drop(columns=to_drop))
di = dict(zip(test['idhogar'], ensamble_predict(models, X_test)+ 1))
sub = test_orig[['Id', 'idhogar']].copy()
sub['Target'] = sub['idhogar'].map(di ).fillna(4)
 sub.drop(columns='idhogar', inplace=True )<save_to_csv>
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'
now = datetime.now().strftime('%y-%m-%d-%H_%M')
 name = f'submission_{now}'
sub = sub.set_index('Id')
 sub['Target'] = sub['Target'].astype('int')
 sub.to_csv(name + '.csv' )<load_from_csv>
embed_size = 50
max_features = 20000
maxlen = 100
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
<categorify>
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
df_combine = pd.concat([train,test], axis = 0)
 id_name = 'Id'
target_name = 'Target'
cols = [f_ for f_ in df_combine.columns if df_combine[f_].dtype == 'object' and f_ != id_name ]
print(cols)
 for col in tqdm(cols):
lb = LabelEncoder()
lb.fit(df_combine[col])
 train[col] = lb.transform(train[col].astype(str))
test[col] = lb.transform(test[col].astype(str))
del lb
gc.collect() 
<feature_engineering>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4h1_percent_in_male'] = df['r4h1'] / df['r4h3']
df['r4m1_percent_in_female'] = df['r4m1'] / df['r4m3']
df['r4h1_percent_in_total'] = df['r4h1'] / df['hhsize']
df['r4m1_percent_in_total'] = df['r4m1'] / df['hhsize']
df['r4t1_percent_in_total'] = df['r4t1'] / df['hhsize']
extract_features(train)
 extract_features(test )<prepare_x_and_y>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))
cols_to_drop = [id_name ,target_name]
X_train = train.drop(cols_to_drop, axis = 1)
 Y_train = train[target_name]
X_train.fillna(0)
 lgb = lgb.LGBMClassifier(class_weight='balanced',drop_rate=0.9, min_data_in_leaf=100, max_bin=255,
n_estimators=500,min_sum_hessian_in_leaf=1,importance_type='gain',learning_rate=0.1,bagging_fraction = 0.85,
colsample_bytree = 1.0,feature_fraction = 0.1,lambda_l1 = 5.0,lambda_l2 = 3.0,max_depth = 9,
min_child_samples = 55,min_child_weight = 5.0,min_split_gain = 0.1,num_leaves = 45,subsample = 0.75)
<train_model>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
k_folds = 5
kf = KFold(n_splits = k_folds, shuffle = True)
 for train_index, test_index in kf.split(X_train, Y_train):
x_train, x_val = X_train.iloc[train_index] , X_train.iloc[test_index]
y_train, y_val = Y_train.iloc[train_index] , Y_train.iloc[test_index]
lgb.fit(x_train, y_train , eval_set= [(x_val, y_val)], early_stopping_rounds = 400 )<predict_on_test>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
X_test = test.drop('Id', axis = 1)
 X_test.fillna(0)
 pres = lgb.predict(X_test )<save_to_csv>
model.fit(X_t, y, batch_size=32, epochs=2, validation_split=0.1);
id_test = test['Id']
subs = pd.DataFrame()
subs['Id'] = id_test
subs['Target'] = pres
subs.to_csv('mean_encoding_target.csv', index = False )<import_modules>
y_test = model.predict([X_te], batch_size=1024, verbose=1 )
!pip install wandb
!pip install bert-tensorflow
print(tf.version.VERSION )<set_options>
y_test[y_test >= 0.5] = 1
y_test[y_test < 0.5] = 0
y_test
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
y_test = y_test.astype(int)
 y_test
SEQUENCE_LENGTH = 128
DATA_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 BERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')
 BERT_PATH_SAVEDMODEL = BERT_PATH + "/bert_multi_from_tfhub"
OUTPUT_PATH = "/kaggle/working"<load_from_csv>
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
y_pseudo_label = np.append(y, y_test, axis = 0)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
list_sentences_train_pseudo = np.append(list_sentences_train, list_sentences_test, axis = 0 )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub2 = pd.read_csv('.. /input/ensemble/submission.csv' )<choose_model_class>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train_pseudo))
list_tokenized_train_pseudo = tokenizer.texts_to_sequences(list_sentences_train_pseudo)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t_pseudo = pad_sequences(list_tokenized_train_pseudo, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )
def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):

 bert_layer = tf.saved_model.load(bert_path)
 bert_layer = hub.KerasLayer(bert_layer, trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
cased = bert_layer.resolved_object.do_lower_case.numpy()
tf.gfile = tf.io.gfile
tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)
 return tokenizer
tokenizer = get_tokenizer()<categorify>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<define_variables>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192<load_pretrained>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer<prepare_x_and_y>
model.fit(X_t_pseudo, y_pseudo_label, batch_size=32, epochs=2, validation_split=0.1);
<define_variables><EOS> 
y_test_final = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test_final
sample_submission.to_csv('submission.csv', index=False )