Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
train_loader, val_loader, test_loader = batch_data(batch_size, X, y, X_test, train_type="classification", val_size=0.2 )<normalization>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
class Clasification(nn.Module):
def __init__(self):
super().__init__()
self.fc1 = nn.Linear(6, 20)
 self.fc2 = nn.Linear(20, 2)
 def forward(self, x):
x = F.relu(self.fc1(x))
x = self.fc2(x)
 return x 
<choose_model_class>
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )
classification_model = Clasification()
classification_model.apply(init_weights)
 classification_model<choose_model_class>
def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
lr = 0.01
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(classification_model.parameters() , lr=lr, momentum=0.9)
 batch_size = 64<train_model>
def build_model_simplelstm(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam' , metrics=['accuracy'])
 return model
def build_model2(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x1),
GlobalMaxPooling1D()(x2),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam' , metrics=['accuracy'])
 return model
train_model(classification_model, batch_size, 3000, criterion )<categorify>
start_time = time.time()
print("Loading data...")
 train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 print("--- %s seconds ---" %(time.time() - start_time))
with torch.no_grad() :
for data in test_loader:
output = classification_model(data.float())
 _, preds = torch.max(output.data, 1)
 survived = preds.numpy()<save_to_csv>
gc.collect()
submission = pd.DataFrame({'PassengerId': test_data['PassengerId'], 'Survived': survived})
 submission.to_csv('submission_classification.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(pd.concat([train['comment_text'], test['comment_text']]), n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
num_train_data = len(train)
 train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))
df = pd.read_csv("submission_classification.csv")
 df.head()<prepare_x_and_y>
del nlp
gc.collect()
X_valid = [next(iter(val_loader)) [0].numpy() ]
y_valid = next(iter(val_loader)) [1].numpy()<import_modules>
start_time = time.time()
print("Build spell correction dictnary...")
 spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/wikinews300d1mvec/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
print("--- %s seconds ---" %(time.time() - start_time))
from xgboost import XGBClassifier
from sklearn.metrics import mean_absolute_error<train_model>
del spell_model
del words
del w_rank
gc.collect()
xg_model = XGBClassifier(learning_rate=0.05, n_estimators=800)
 xg_model.fit(X, y )<predict_on_test>
start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 print("--- %s seconds ---" %(time.time() - start_time))
predictions = xg_model.predict(X_test)
 predictions<save_to_csv>
del word_dict
del lemma_dict
gc.collect()
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('xg_boost_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
max_length = 220
embedding_size = 300
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 print(train_word_sequences[:1])
 print(test_word_sequences[:1])
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32 )
warnings.filterwarnings('ignore' )<load_from_csv>
gc.collect()
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv")
 display(test.head(15))
display(train.describe() )<feature_engineering>
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train[column] = np.where(train[column] >= 0.5, True, False)
 sample_weights = np.ones(len(train), dtype=np.float32)
 sample_weights += np.array(train[IDENTITY_COLUMNS].sum(axis=1))
sample_weights += train[TARGET_COLUMN] *(~train[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train[TARGET_COLUMN])* train[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()
all_data['Docker_num'] = [cab[:1] if pd.notnull(cab)else "Unknown" for cab in all_data['Cabin']]
all_data['Has_cabin_informed'] = [1 if pd.notnull(cab)else 0 for cab in all_data['Cabin']]
all_data['Title'] = [re.search('\, (.*)\.', name ).group(1)for name in all_data['Name']]
all_data.set_value(all_data['PassengerId']==514, 'Title', ' Mrs')
 dictionary_of_titles = {
" Capt": "Crew",
" Col": "Crew",
" Major": "Crew",
" Dr": "Crew",
" Rev": "Crew",
" Jonkheer": "VIP",
" Don": "VIP",
" Dona": "VIP",
" Sir" : "VIP",
" the Countess":"VIP",
" Lady" : "VIP",
" Mme": "Mrs",
" Ms": "Miss",
" Mrs" : "Mrs",
" Mlle": "Miss",
" Miss" : "Miss",
" Mr" : "Mr",
" Master" : "Master"
}
all_data['Title'] = all_data.Title.map(dictionary_of_titles)
 def fill_missing_age_title(all_data):
class_age_female_miss = all_data[(all_data['Title']=='Miss')]['Age'].dropna().mean()
class_age_female_mrs = all_data[(all_data['Title']=='Mrs')]['Age'].dropna().mean()
class_age_male_master = all_data[(all_data['Title']=='Master')]['Age'].dropna().mean()
class_age_male_mr = all_data[(all_data['Title']=='Mr')]['Age'].dropna().mean()
all_data[all_data['Age'].isnull() ==True].head(5)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Miss')].index, 'Age', class_age_female_miss)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Mrs')].index, 'Age', class_age_female_mrs)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Master')].index, 'Age', class_age_male_master)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Mr')].index, 'Age', class_age_male_mr)
 return all_data
all_data = fill_missing_age_title(all_data)
 def age_cat(age):
if age <= 15:
return 1
if age <= 35:
return 2
if age <= 55:
return 3
if age > 55:
return 4
else:
return 0
all_data['Age_cat'] = all_data['Age'].apply(age_cat)
 def fare_cat(fare):
if fare < 15:
return 1
if fare < 35:
return 2
if fare < 100:
return 3
if fare > 100:
return 4
else:
return 0
all_data['Fare_cat'] = all_data['Age'].apply(fare_cat)
 all_data['Relatives'] = all_data['SibSp'] + all_data['Parch']
def fill_families(all_data):
all_data['Last_Name'] = all_data['Name'].apply(lambda x: str.split(x, ",")[0])
 default_survival_chance = 0.5
all_data['Family_Survival'] = default_survival_chance
for grp, grp_df in all_data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
all_data['IsAlone'] = 0
all_data.loc[all_data['Relatives'] == 0, 'IsAlone'] = 1
return all_data
all_data = fill_families(all_data)
 all_data.head(5 )<feature_engineering>
y_train = train[TARGET_COLUMN].values
y_aux_train = train[AUX_COLUMNS].values
sns.distplot(all_data['Fare'].dropna())
 plt.ylabel('Frequency')
 plt.title('Fare distribution')
 all_data['Fare']=all_data['Fare'].apply(lambda x: np.log(x))<feature_engineering>
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
sns.distplot(all_data['Age'].dropna())
 plt.ylabel('Frequency')
 plt.title('Age distribution')
 all_data['Age']=all_data['Age'].apply(lambda x: np.log(x))
print("Feature engineering: Completed" )<split>
start_time = time.time()
print("Start training...")
 checkpoint_predictions = []
weights = []
for model_idx in range(1):
model = build_model_simplelstm(embedding_matrix_glove, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 train_word_sequences,
[y_train, y_aux_train],
batch_size=400,
epochs=1,
verbose=1,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(test_word_sequences, batch_size=2048)[0].flatten())
 result_ = pd.DataFrame.from_dict({
'id': test.id,
'prediction': checkpoint_predictions[-1]
})
 result_.to_csv("model_%d_%d.csv" %(model_idx, global_epoch))
weights.append(2 ** global_epoch)
 print("--- %s seconds ---" %(time.time() - start_time))
X = all_data[:len(train)]
X_test_full = all_data[len(train):]
y = X.Survived
X.drop('Survived', axis=1, inplace=True)
 print(len(all_data), len(X), len(X_test_full))
X_train_full, X_valid_full, y_train, y_valid = train_test_split(X, y, train_size=0.95, test_size=0.05, random_state=0)
 low_cardinality_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and
X_train_full[cname].dtype not in ['int64', 'float64']]
print("Low cardinality columns: ", low_cardinality_cols)
 numeric_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]
print("Numeric columns: ", numeric_cols)
 my_cols = low_cardinality_cols + numeric_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()
X_train = pd.get_dummies(X_train)
 X_valid = pd.get_dummies(X_valid)
 X_test = pd.get_dummies(X_test)
 X_train, X_valid = X_train.align(X_valid, join='left', axis=1)
 X_train, X_test = X_train.align(X_test, join='left', axis=1 )<train_model>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )
<train_model><EOS>