Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
test = train_df[train_df_len:]
test.drop(labels = ["Survived"],axis = 1, inplace = True)
<split>
ps = PorterStemmer()
ls = LancasterStemmer()
ss = SnowballStemmer('english' )
train = train_df[:train_df_len]
X_train = train.drop(labels = "Survived", axis = 1)
 y_train = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.50, random_state = 42)
 print("X_train",len(X_train))
print("X_test",len(X_test))
print("y_train",len(y_train))
print("y_test",len(y_test))
print("test",len(test))<train_model>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 acc_log_train = round(logreg.score(X_train, y_train)*100,2)
 acc_log_test = round(logreg.score(X_test,y_test)*100,2)
 print("Training Accuracy: % {}".format(acc_log_train))
print("Testing Accuracy: % {}".format(acc_log_test))<choose_model_class>
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white',
'psychiatric_or_mental_illness']
for col in identity_columns+['target']:
train[col] = np.where(train[col]>=0.5, True, False )
random_state = 42
classifier = [DecisionTreeClassifier(random_state = random_state),
SVC(random_state = random_state),
RandomForestClassifier(random_state = random_state),
LogisticRegression(random_state = random_state),
KNeighborsClassifier() ]
dt_param_grid = {"min_samples_split" : range(10,500,20),
"max_depth": range(1,20,2)}
svc_param_grid = {"kernel" : ["rbf"],
"gamma": [0.001, 0.01, 0.1, 1],
"C": [1,10,50,100,200,300,1000]}
rf_param_grid = {"max_features": [1,3,10],
"min_samples_split":[2,3,10],
"min_samples_leaf":[1,3,10],
"bootstrap":[False],
"n_estimators":[100,300],
"criterion":["gini"]}
logreg_param_grid = {"C":np.logspace(-3,3,7),
"penalty": ["l1","l2"]}
knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int ).tolist() ,
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
classifier_param = [dt_param_grid,
svc_param_grid,
rf_param_grid,
logreg_param_grid,
knn_param_grid]<train_on_grid>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
set_random_seed(2)
 seed_everything()
cv_result = []
best_estimators = []
for i in range(len(classifier)) :
clf = GridSearchCV(classifier[i], param_grid=classifier_param[i], cv = StratifiedKFold(n_splits = 10), scoring = "accuracy", n_jobs = -1,verbose = 1)
 clf.fit(X_train,y_train)
 cv_result.append(clf.best_score_)
 best_estimators.append(clf.best_estimator_)
 print(cv_result[i] )<train_model>
max_features = 150000
maxlen = 256
embed_size = 302
votingC = VotingClassifier(estimators = [("dt",best_estimators[0]), 
("rfc",best_estimators[2]), 
("lr",best_estimators[3])],
voting = "soft", n_jobs = -1)
 votingC = votingC.fit(X_train, y_train)
 print(accuracy_score(votingC.predict(X_test),y_test))<save_to_csv>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
test_survived = pd.Series(votingC.predict(test), name = "Survived" ).astype(int)
 results = pd.concat([test_PassengerId, test_survived],axis = 1)
 results.to_csv("titanic.csv", index = False )<load_from_csv>
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
submission_format = pd.read_csv("/kaggle/input/titanic/gender_submission.csv" )<load_from_csv>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
train_data_orig = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data=train_data_orig.copy()
train_data.head()<load_from_csv>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
test_data_orig = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data=test_data_orig.copy()
test_data.head()<count_unique_values>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
distinct_classes,value_counts=np.unique(train_data.Survived,return_counts=True )<count_missing_values>
def load_and_proc() :
train['comment_text'] = train['comment_text'].apply(lambda x : clean_text(x))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_text(x))
train['comment_text'] = train['comment_text'].apply(lambda x : clean_numbers(x))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_numbers(x))
train['comment_text'] = train['comment_text'].apply(lambda x : clean_contractions(x,contraction_mapping))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_contractions(x,contraction_mapping))
df_train, df_valid = train_test_split(train, test_size=0.33)
 df_test = test
df_train.loc[:,'set_'] = 'train'
df_valid.loc[:,'set_'] = 'valid'
df_test.loc[:,'set_'] = 'test'
set_indices = df_train.loc[:,'set_']
set_indices.append(df_valid.loc[:,'set_'])
 set_indices.append(df_test.loc[:,'set_'])
 y_train = np.asarray(df_train['target'])
 y_valid = np.asarray(df_valid['target'])
 set_indices_label = df_train.loc[:,'set_']
set_indices_label = set_indices_label.append(df_valid.loc[:,'set_'])
 X_train = df_train['comment_text'].fillna('_
X_val = df_valid['comment_text'].fillna('_
X_test = df_test['comment_text'].fillna('_
all_text = list(X_train)
 all_text.append(list(X_val))
all_text.append(list(X_test))
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(all_text)
 X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=maxlen)
 X_val = pad_sequences(X_val, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 np.random.seed(2019)
 train_idx = np.random.permutation(len(X_train))
valid_idx = np.random.permutation(len(X_val))
print(type(X_train))
print(type(y_train))
X_train = X_train[train_idx]
X_val = X_val[valid_idx]
Y_train = y_train[train_idx]
Y_val = y_valid[valid_idx]
return X_train, X_val, X_test, Y_train, Y_val, tokenizer.word_index
train_data.isnull().sum()<count_missing_values>
X_train, X_val,X_test, Y_train, Y_val, word_index = load_and_proc()
test_data.isnull().sum()<drop_column>
gc.collect()
del train
del test
train_data.drop('Cabin',axis=1,inplace=True)
 test_data.drop('Cabin',axis=1,inplace=True )<count_missing_values>
def load_glove() :
EMBEDDING_FILE = '.. /input/emb-model/glove.840B.300d.txt'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)[:300]
embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE))
return embeddings_index
train_data.isnull().sum()<count_missing_values>
def load_paragram() :
EMBEDDING_FILE = '.. /input/paragram-300-sl999/paragram_300_sl999.txt'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)[:300]
embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE, errors='ignore', encoding='utf-8'))
return embeddings_index
test_data.isnull().sum()<drop_column>
def load_fasttext() :
EMBEDDING_FILE = '.. /input/fasttext-wiki-news-300d-1m/wiki-news-300d-1M.vec'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)
 embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)> 100)
 return embeddings_index
train_data.drop('PassengerId',axis=1,inplace=True )<drop_column>
def create_emb(word_index, embeddings_index):
emb_mean,emb_std = -0.005838499,0.48782197
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
count_found = nb_words
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 word_sent = TextBlob(word ).sentiment
extra_embed = [word_sent.polarity, word_sent.subjectivity]
if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.lower()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.upper()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.capitalize()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ps.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ls.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ss.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
embedding_matrix[i,300:] = extra_embed
count_found -= 1
print('Total words ', nb_words)
 print("Got embedding for ",count_found," words.")
 return embedding_matrix
testPIds=test_data['PassengerId']
test_data.drop('PassengerId',axis=1,inplace=True )<feature_engineering>
final_matrix = glove_embedding
def change_gender_num(s):
if(s=='male'):
return 1
else:
return 0
train_data['Sex']=train_data['Sex'].apply(change_gender_num)
 test_data['Sex']=test_data['Sex'].apply(change_gender_num )<drop_column>
gc.collect()
train_data['relatives']=train_data['SibSp']+train_data['Parch']+1
train_data.drop(['SibSp','Parch'],axis=1,inplace=True )<drop_column>
def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel )
test_data['relatives']=test_data['SibSp']+test_data['Parch']+1
test_data.drop(['SibSp','Parch'],axis=1,inplace=True )<count_unique_values>
class Attention(Layer):
def __init__(self,
W_regulariser=None,b_regulariser=None,u_regulariser=None,
W_constraint=None,b_constraint=None,u_constraint=None,
biase=True, **kwargs):
self.support_masking = True
self.initializer = initializers.get('glorot_uniform')
 self.W_regulariser = regularizers.get(W_regulariser)
 self.b_regulariser = regularizers.get(b_regulariser)
 self.u_regulariser = regularizers.get(u_regulariser)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.u_constraint = constraints.get(u_constraint)
 self.biase = biase
super(Attention,self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[-1],input_shape[-1],),
initializer=self.initializer,
name='{}_W'.format(self.name),
regularizer=self.W_regulariser,
constraint=self.W_constraint)
 if self.biase:
self.b = self.add_weight(shape=(input_shape[-1],),
initializer = 'zero',
name='{}_b'.format(self.name),
regularizer=self.b_regulariser,
constraint=self.b_constraint 
)
 self.u = self.add_weight(shape=(input_shape[-1],),
initializer=self.initializer,
name='{}_u'.format(self.name),
regularizer=self.u_regulariser,
constraint=self.u_constraint)
 super(Attention, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
 if self.biase:
uit += self.b
uit = K.tanh(uit)
 ait = dot_product(uit, self.u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1]
len(train_data.Ticket.unique() )<drop_column>
def model_lstm_du(final_embedding):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[final_embedding], trainable = False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x1)
 x1 = Conv1D(64, kernel_size=3, padding = "valid", kernel_initializer = "he_uniform" )(x1)
 x2 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x2)
 x2 = Conv1D(64, kernel_size=4, padding = "valid", kernel_initializer = "he_uniform" )(x2)
 avg_pool1 = GlobalAveragePooling1D()(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 avg_pool2 = GlobalAveragePooling1D()(x2)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = concatenate([avg_pool1, max_pool1,avg_pool2, max_pool2])
 conc = Dense(128, activation='relu' )(conc)
 conc = Dense(64, activation='relu' )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs = inp, outputs = outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr = 1e-3, decay=0.0), metrics=['accuracy'])
 return model
train_data.drop('Ticket',axis=1,inplace=True )<drop_column>
gc.collect()
test_data.drop('Ticket',axis=1,inplace=True )<feature_engineering>
def train_pred(model, epoch=2):
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, verbose=1,patience=1, mode='min')
 callbacks = [checkpoint,early_stopping]
model.fit(x=X_train,y=Y_train, batch_size=512,epochs=epoch, callbacks=callbacks, validation_data=(X_val, Y_val))
model.load_weights(filepath)
 pred_val_y = model.predict([X_val], batch_size=1024, verbose=0)
 pred_test_y = model.predict([X_test], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
def get_title(name):
return name.strip().split(',')[1].split('.')[0]
train_data['title']=train_data['Name'].apply(get_title)
 train_data.drop('Name',axis=1,inplace=True)
 test_data['title']=test_data['Name'].apply(get_title)
 test_data.drop('Name',axis=1,inplace=True )<set_options>
pred_val_y, pred_test_y = train_pred(model, epoch=8 )
matplotlib.rcParams['figure.figsize']=(15,10 )<count_unique_values>
submission = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv")
 submission['prediction'] = pred_test_y
submission.to_csv('submission.csv', index=False )