Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<categorify>
print(os.listdir(".. /input"))
print(os.listdir(".. /input/glove840b300dtxt"))
print(os.listdir(".. /input/fasttext-crawl-300d-2m"))
print(os.listdir(".. /input/jigsaw-unintended-bias-in-toxicity-classification"))
vectorizer = TfidfVectorizer(max_features=30,stop_words='english')
 all_tweets=list(train['text'])+list(test['text'])
 X = vectorizer.fit_transform(all_tweets)
 tweet_array=X.toarray()
tf_train=tweet_array[0:len(train)]
tf_test=tweet_array[len(train):len(tweet_array)]
train_df=np.concatenate([X_train_embeddings['outputs'],tf_train],axis=1)
 test_df=np.concatenate([X_test_embeddings['outputs'],tf_test],axis=1 )<train_model>
SEED = 1234
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 8
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 1
MAX_LEN = 220
DROPOUT_RATE = 0.3
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '""“”’`'!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
text_clf = lgb.LGBMClassifier(n_estimators=3000, learning_rate=0.05)
 text_clf.fit(train_df, target )<save_to_csv>
CONTRACTIONS = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is",
"how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will",
"I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have",
"i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
"oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have",
"that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would",
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have",
"wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have",
"when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will",
"who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have",
"you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", "it's": "it is" }
pred=text_clf.predict(test_df)
 ssub["target"] = pred
ssub.to_csv("submission.csv",index=False )<set_options>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 def clean_contractions(str, mapping):

 specials = ["’", "‘", "´", "`"]
for s in specials:
str = str.replace(s, "'")
 res = ' '.join([mapping[t] if t in mapping else t for t in str.split(' ')])
 return res
def clean_special_chars(text, punct, replacement=' '):

 for p in punct:
text = text.replace(p, replacement)
 return text
def preprocess(data):
data = data.astype(str)
 data = data.apply(lambda x: clean_contractions(x, mapping=CONTRACTIONS))
data = data.apply(lambda x: clean_special_chars(x, punct=CHARS_TO_REMOVE))
return data
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(DROPOUT_RATE )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
warnings.filterwarnings('ignore')
 sns.set_style('whitegrid')
 %matplotlib inline<load_from_csv>
seed_everything(SEED)
 train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train_df.shape, test_df.shape
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
%%time
x_train = preprocess(train_df[TEXT_COLUMN])
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = preprocess(test_df[TEXT_COLUMN])
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )
for col in train.columns:
print('number of null values in ' + col + ': ' + str(train[pd.isnull(train[col])].shape[0]))<count_missing_values>
%%time
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()
print(sample_weights.shape )
for col in test.columns:
print('number of null values in ' + col + ': ' + str(test[pd.isnull(test[col])].shape[0]))<categorify>
%%time
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 print(embedding_matrix.shape)
train = train.join(pd.get_dummies(train.Sex))
test = test.join(pd.get_dummies(test.Sex))<categorify>
%%time
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
seed_everything(SEED + model_idx)
 model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
print('model {} epoch {}'.format(model_idx + 1, global_epoch + 1))
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
class_dummies = pd.get_dummies(train.Pclass)
 class_dummies.columns = ['Higher', 'Middle', 'Lower']
train = train.join(class_dummies )<prepare_x_and_y>
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.head()
X_train = train[['male', 'female','Higher', 'Middle', 'Lower']]
y = train['Survived']
class_dummies = pd.get_dummies(test.Pclass)
 class_dummies.columns = ['Higher', 'Middle', 'Lower']
test = test.join(class_dummies)
 X_test = test[['male', 'female', 'Higher', 'Middle', 'Lower']]<save_to_csv>
submission.to_csv('submission.csv', index=False)
 print(os.listdir("."))