Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<compute_train_metric>
import gc
import numpy as np
import pandas as pd
from typing import List, Tuple, Dict
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
scores = cross_val_score(my_pipeline, X_train, y_train,
cv=5,
scoring='accuracy')
 print(scores )<save_to_csv>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
EPOCHS = 4
BATCH_SIZE = 256
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
output = pd.DataFrame({'PassengerId': X_test.index+892,
'Survived': preds_test.astype(int)})
 output.to_csv('submission.csv', index=False )<set_options>
def get_coefs(word: str, *arr: str)->(str, np.ndarray):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path: str)-> Dict[str, np.ndarray]:

 with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index: Dict[str, int], path: str, indexesOfWordsContainTrump: List[int])-> np.ndarray:

 embedding_dict = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
if(i in indexesOfWordsContainTrump):
embedding_matrix[i] = embedding_dict['Trump']
else:
try:
embedding_matrix[i] = embedding_dict[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix: np.ndarray)-> Model:

 words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(len(AUX_COLUMNS), activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
warnings.filterwarnings('ignore' )<load_from_csv>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False)
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()<drop_column>
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] &(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])& train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN] & train_df['homosexual_gay_or_lesbian'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['black'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['white'] + 0)* 5
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['muslim'] + 0)* 1
sample_weights +=(~train_df[TARGET_COLUMN] & train_df['jewish'] + 0)* 1
sample_weights /= sample_weights.mean()
train = train.drop(train[(train['GarageCars']>3)&(train['SalePrice']<300000)].index ).reset_index(drop=True )<concatenate>
indexesOfWordsContainTrump = []
for word, index in tokenizer.word_index.items() :
if(( 'trump' in word)or('Trump' in word)) :
indexesOfWordsContainTrump.append(index)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, filePath, indexesOfWordsContainTrump)for filePath in EMBEDDING_FILES], axis=-1 )
y_train = train.SalePrice
join_data = pd.concat(( train, test), sort=False ).reset_index(drop=True)
 train_size = len(train)
 test_size = len(test)
 print("Train set size: ", train_size)
 print("Test set size: ", test_size)
 print("Train+test set size: ", len(join_data))<create_dataframe>
del train_df
del tokenizer
gc.collect()
missings_count = {col:join_data[col].isnull().sum() for col in join_data.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))<categorify>
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.6 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )
<feature_engineering><EOS> 
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )