Kaggle competition: Toxic Comment Classification Challenge
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from tensorflow import keras
from keras import layers
from keras.callbacks import Callback
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from textblob import TextBlob
from textblob import Word
import re
import string
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"] )<categorify>
!unzip -q "/kaggle/input/jigsaw-toxic-comment-classification-challenge/*.zip"
!dir
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
train_data_file = "train.csv"
test_data_file = "test.csv"
submission_file = "sample_submission.csv"
train_data = pd.read_csv(train_data_file)
 test_data = pd.read_csv(test_data_file)
 submission_result = pd.read_csv(submission_file )
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
max_len = 120
embedding_dim = 300
vocabulary_size = 20000
num_tokens = vocabulary_size+1
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
def preprocess(corpus):
printable = set(string.printable)
 corpus = ''.join(filter(lambda x: x in printable, corpus))
corpus = corpus.lower()
corpus = re.sub(r"won't", "will not", corpus)
 corpus = re.sub(r"can't", "can not", corpus)
 corpus = re.sub(r"ain't","is not", corpus)
 corpus = re.sub(r"shan't", "shall not", corpus)
 corputs = re.sub(r"let's", "let us", corpus)
 corpus = re.sub(r"n't", " not", corpus)
 corpus = re.sub(r"'re", " are", corpus)
 corpus = re.sub(r"'s", " is", corpus)
 corpus = re.sub(r"'d", " would", corpus)
 corpus = re.sub(r"'ll", " will", corpus)
 corpus = re.sub(r"'t", " not", corpus)
 corpus = re.sub(r"'ve", " have", corpus)
 corpus = re.sub(r"'m", " am", corpus)
 corpus = re.sub(r"'", " ", corpus)
 correction_list = {"youfuck": "you fuck", \
"fucksex": "fuck sex",\
"bitchbot": "bitch bot",\
"offfuck": "fuck off",\
"donkeysex": "donkey sex",\
"securityfuck": "security fuck",\
"ancestryfuck": "ancestry fuck",\
"turkeyfuck": "turkey fuck",\
"faggotgay": "faggot gay",\
"fuckbot": "fuck bot",\
"assfuckers": "ass fucker",\
"ckckck": "cock",\
"fuckfuck": "fuck",\
"lolol": "lol",\
"pussyfuck": "fuck",\
"gaygay": "gay",\
"haha": "ha",\
"sucksuck": "suck"
}
for old,new in correction_list.items() :
corpus = corpus.replace(old,new)
 return corpus
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
tokenizer = Tokenizer(num_words = vocabulary_size+1,\
filters='!"
0123456789',\
lower=True, split=' ')
 X_train_raw = train_data["comment_text"]
X_test_raw = test_data["comment_text"]
bad_comment_cat = ['toxic', 'severe_toxic', 'obscene', 'threat',\
'insult', 'identity_hate']
Y_train = train_data[bad_comment_cat]
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<import_modules>
X_train_raw = X_train_raw.apply(lambda x: preprocess(str(x)))
 X_test_raw = X_test_raw.apply(lambda x: preprocess(str(x)))
 tokenizer.fit_on_texts(X_train_raw)
 tokenizer.fit_on_texts(X_test_raw)
 X_train = pad_sequences(tokenizer.texts_to_sequences(X_train_raw),\
maxlen = max_len, truncating = "pre")
 X_test = pad_sequences(tokenizer.texts_to_sequences(X_test_raw),\
maxlen = max_len, truncating = "pre")
 x_train, x_val, y_train, y_val = train_test_split(X_train,Y_train,train_size=0.9, random_state=199)
 print(x_train.shape," ",y_train.shape," ",x_val.shape," ",y_val.shape )
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import time<load_from_csv>
def get_weights(embedding_vectors,embedding_dim):
global num_tokens,tokenizer
embedding_weights = np.zeros(( num_tokens,embedding_dim))
misses = 0
for word, i in tokenizer.word_index.items() :
vector = embedding_vectors.get(word)
 if i>=num_tokens :
break
elif vector is not None:
embedding_weights[i] = vector
else:
if len(word)<20:
word = Word(word)
 word = word.spellcheck() [0][0]
vector = embedding_vectors.get(str(word))
if vector is not None:
embedding_weights[i] = vector
else:
misses +=1
else:
misses +=1
print(f"The number of missed words is {misses}")
 return embedding_weights
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = train_target_df.columns[1:]
N_TARGETS = len(target_cols)
 print(train_df.shape )<define_variables>
embedding_vectors_fasttext = {}
with open("/kaggle/input/fasttext-crawl-300d-2m/crawl-300d-2M.vec","r")as file:
file.readline()
for line in file:
word , vector = line.split(maxsplit=1)
 vector = np.fromstring(vector,"float32",sep=" ")
 embedding_vectors_fasttext[word] = vector
GENES = [col for col in train_df.columns if col.startswith('g-')]
CELLS = [col for col in train_df.columns if col.startswith('c-')]<concatenate>
embedding_weights_fasttext = get_weights(embedding_vectors_fasttext,embedding_dim=300 )
n_comp = 50
data = pd.concat([pd.DataFrame(train_df[GENES]), pd.DataFrame(test_df[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_df.shape[0]]; test2 = data2[-test_df.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_df = pd.concat(( train_df, train2), axis=1)
 test_df = pd.concat(( test_df, test2), axis=1 )<concatenate>
embedding_vectors_glove = {}
with open("/kaggle/input/glove6b/glove.6B.300d.txt","r")as file:
for line in file:
word , vector = line.split(maxsplit=1)
 vector = np.fromstring(vector,"float32",sep=" ")
 embedding_vectors_glove[word] = vector
n_comp = 15
data = pd.concat([pd.DataFrame(train_df[CELLS]), pd.DataFrame(test_df[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_df.shape[0]]; test2 = data2[-test_df.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_df = pd.concat(( train_df, train2), axis=1)
 test_df = pd.concat(( test_df, test2), axis=1 )<prepare_x_and_y>
embedding_weights_glove = get_weights(embedding_vectors_glove,embedding_dim=300 )
train_copy = train_df
var_thresh = VarianceThreshold(0.8)
 data = train_df.append(test_df)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_df_transformed = data_transformed[ : train_df.shape[0]]
test_df_transformed = data_transformed[-test_df.shape[0] : ]
train_df = pd.DataFrame(train_df[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_df = pd.concat([train_df, pd.DataFrame(train_df_transformed)], axis=1)
 test_df = pd.DataFrame(test_df[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_df = pd.concat([test_df, pd.DataFrame(test_df_transformed)], axis=1)
 train_df.shape<rename_columns>
def GRU_model_glove() :
global max_len,num_tokens,embedding_weights_glove
inputs = layers.Input(shape=(max_len,))
x = layers.Embedding(input_dim=num_tokens,\
output_dim=embedding_dim,\
embeddings_initializer=keras.initializers.Constant(embedding_weights_glove),\
trainable=True )(inputs)
 x = layers.SpatialDropout1D(0.3 )(x)
 forward_layer = layers.GRU(42,return_sequences=True)
 backward_layer = layers.GRU(42,activation="relu",dropout=0.1,return_sequences=True,go_backwards=True)
 x = layers.Bidirectional(forward_layer,backward_layer=backward_layer )(x)
 x = layers.GlobalMaxPooling1D()(x)
 outputs = layers.Dense(units=6,activation='sigmoid' )(x)
 model = keras.models.Model(inputs=inputs, outputs=outputs, name="GRU_model_glove")
 model.compile(optimizer=tf.optimizers.Adam() ,\
loss=tf.losses.BinaryCrossentropy() ,\
metrics=['AUC'])
 return model
GRU_model_glove = GRU_model_glove()
GRU_model_glove.summary()
search_row = dict(train_copy.iloc[0,4:])
 col_rela = {}
for i in np.arange(0,868):
for k, v in search_row.items() :
if train_df[i][0] == v:
col_rela[i] = k
train_df = train_df.rename(columns=col_rela)
 test_df = test_df.rename(columns=col_rela )<define_variables>
history = GRU_model_glove.fit(x_train, y_train, epochs=2,\
batch_size=128, validation_data=(x_val,y_val))
SEED = 1234
EPOCHS = 28
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.0005
N_TARGETS = len(target_cols )<define_variables>
def GRU_model_fasttext() :
global max_len,num_tokens,embedding_weights_fasttext
inputs = layers.Input(shape=(max_len,))
x = layers.Embedding(input_dim=num_tokens,\
output_dim=embedding_dim,\
embeddings_initializer=keras.initializers.Constant(embedding_weights_fasttext),\
trainable=False )(inputs)
 x = layers.SpatialDropout1D(0.3 )(x)
 forward_layer = layers.GRU(64,return_sequences=True)
 backward_layer = layers.GRU(64,activation="relu",dropout=0.3,return_sequences=True,go_backwards=True)
 x = layers.Bidirectional(forward_layer,backward_layer=backward_layer )(x)
 avg_pool = layers.GlobalAveragePooling1D()(x)
 max_pool = layers.GlobalMaxPooling1D()(x)
 x = layers.concatenate([avg_pool,max_pool])
 outputs = layers.Dense(units=6,activation='sigmoid' )(x)
 model = keras.models.Model(inputs=inputs, outputs=outputs, name="GRU_model")
 model.compile(optimizer=tf.optimizers.Adam() ,\
loss=tf.losses.BinaryCrossentropy() ,\
metrics=['AUC'])
 return model
GRU_model_fasttext = GRU_model_fasttext()
GRU_model_fasttext.summary()
history = GRU_model_fasttext.fit(x_train, y_train, \
epochs=2, batch_size=32,\
validation_data=(x_val,y_val))
def seed_everything(seed):
np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<compute_test_metric>
model_nums = 2
size1 = x_train.shape[0]
y_train_pred = np.zeros(( model_nums,size1,6),dtype="float32")
 y_train_pred[0] = GRU_model_fasttext.predict(x_train)
 y_train_pred[1] = GRU_model_glove.predict(x_train)
 size2 = X_test.shape[0]
y_test_pred = np.zeros(( model_nums,size2,6),dtype="float32")
 y_test_pred[0] = GRU_model_fasttext.predict(X_test)
 y_test_pred[1] = GRU_model_glove.predict(X_test)
 y_pred = np.zeros(( size2,6),dtype="float32")
 for i in range(6):
lg = LogisticRegression()
temp = np.zeros(( size1,model_nums),dtype="float32")
 for j in range(model_nums):
temp[:,j] = y_train_pred[j,:,i]
lg.fit(temp,y_train[bad_comment_cat[i]])
 temp = np.zeros(( size2,model_nums),dtype="float32")
 for j in range(model_nums):
temp[:,j] = y_test_pred[j,:,i]
y_pred[:,i] = lg.predict_proba(temp)[:,1]
<data_type_conversions><EOS> 
submission_result[bad_comment_cat] = y_pred
submission_result.to_csv("submission.csv",index=False )