Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
resultats = comparaisonsClassifieurs(classifieursArbresDict, X_train, X_test, y_train, y_test )<choose_model_class>
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
import torch
from torch.utils import data
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
import time
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from fastai.basics import *
from fastai.basic_train import Learner
from fastai.callbacks.general_sched import *
import gc
classifieur = CatBoostClassifier(iterations=250, depth=3, silent=True )<compute_test_metric>
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
resultats = controleClassifieur(classifieur, X_train, X_test, y_train, y_test )<concatenate>
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
resultatsFinaux,classifieursCV = effectueValidationCroisee(classifieur,
X,
y,
n_splits = 15 )<prepare_output>
DROPOUT_RATE = 0.2
EPOCHS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
output = pd.DataFrame({'PassengerId':test.index, 'Survived':np.zeros(test.shape[0])})
 for i in range(len(classifieursCV)) :
output.Survived += classifieursCV[i].predict_proba(test)[:,1]
output.Survived /= len(classifieursCV)
 output.Survived = output.Survived.round().astype('int8')
 output.head()<save_to_csv>
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
output.to_csv('submission008.csv',index=False )<set_options>
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, 1, 0 )
warnings.filterwarnings('ignore', category = DeprecationWarning)
 warnings.filterwarnings('ignore', category = FutureWarning)
 sns.set(style='white', context='notebook', palette='deep')
 plt.style.use('bmh')
 train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 IDtest = test["PassengerId"]
train.shape<feature_engineering>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values[:,np.newaxis]
x_test = test_df[TEXT_COLUMN].astype(str )
def IQR_outlier(df,n,features):
outlier_indices=[]
for col in features:
Q1=np.percentile(df[col],25)
 Q3=np.percentile(df[col],75)
 IQR=Q3-Q1
outlier_step=1.5*IQR
outlier_list_col=df[(df[col]<Q1-outlier_step)|(df[col]>Q3+outlier_step)].index
print('Total Number Outliers of',col,' : ',len(outlier_list_col))
print('Percentage of Outliers of',col,' : ',np.round(len(outlier_list_col)/len(df[col])*100),'%')
 outlier_indices.extend(outlier_list_col)
 outlier_indices=Counter(outlier_indices)
 multiple_outliers=list(k for k, v in outlier_indices.items() if v>n)
 return multiple_outliers
num_features=['Age','SibSp','Parch','Fare']
Outliers_to_drop=IQR_outlier(train,2,num_features)
 print('Total number of Outlier indices : ', len(Outliers_to_drop)) 
<filter>
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
train.loc[Outliers_to_drop,num_features]<count_missing_values>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )
dataset["Fare"].isnull().sum()<feature_engineering>
batch_size = 1024
dataset["Fare"]=dataset["Fare"].fillna(dataset["Fare"].median() )<feature_engineering>
y_train_torch = torch.from_numpy(y_train)
 y_train_torch = y_train_torch.float()
dataset["Fare"] = dataset["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )<categorify>
train_lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
dataset['Sex']=dataset['Sex'].map({'male':0,'female':1} )<count_missing_values>
maxlen = train_lengths.max()
print('Number of Null entries: ',dataset['Embarked'].isnull().sum())
 print('Most common dock: ',dataset.Embarked.mode() [0] )<count_missing_values>
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
print("x_train_padded size:")
 print(x_train_padded.shape)
 x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))
print("x_test_padded size:")
 print(x_test_padded.shape)
 del x_train, x_test
display(dataset.Cabin.shape)
 print('Number of null values : ', dataset.Cabin.isnull().sum())
 print('Percentage of null values : ', round(dataset.Cabin.isnull().sum() /dataset.Cabin.shape[0]*100),'%' )<count_missing_values>
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None, weight_index = None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.weight_index = weight_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i not in [self.label_index,self.weight_index]],[batch[self.label_index],batch[self.weight_index]]
return batch
dataset.isnull().sum()<feature_engineering>
train_df[TARGET_COLUMN] = train_df[TARGET_COLUMN].astype(np.bool8)
 train_df[IDENTITY_COLUMNS] = train_df[IDENTITY_COLUMNS].astype(np.bool8)
 sample_weights = np.ones(len(x_train_padded), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1 ).values
sample_weights += np.abs(train_df[TARGET_COLUMN] * \ 
(~train_df[IDENTITY_COLUMNS] ).sum(axis=1 ).values)
 sample_weights += np.abs(train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1 ).values)
 sample_weights += np.abs(~train_df[TARGET_COLUMN])* \
train_df[IDENTITY_COLUMNS].sum(axis=1 ).values * 5
sample_weights /= sample_weights.mean()
sample_weights = torch.from_numpy(sample_weights.values[:,np.newaxis] ).float()
index_NaN_age = dataset["Age"][dataset.Age.isnull() ].index
for i in index_NaN_age :
age_med = dataset["Age"].median()
age_pred = dataset["Age"][(( dataset['SibSp'] == dataset['SibSp'][i])&(dataset['Parch'] == dataset['Parch'][i])&(dataset['Pclass'] == dataset['Pclass'][i])) ].median()
if not np.isnan(age_pred):
dataset['Age'].iloc[i] = age_pred
else :
dataset['Age'].iloc[i] = age_med 
<count_missing_values>
train_dataset = data.TensorDataset(x_train_padded, train_lengths, y_train_torch, sample_weights )
dataset.isnull().sum()<groupby>
test_dataset = data.TensorDataset(x_test_padded, test_lengths )
print('Mean : ', dataset[['Cabin','Survived']].groupby('Cabin' ).mean())
 print('Count : ', dataset[['Cabin','Survived']].groupby('Cabin' ).count() )<feature_engineering>
valid_dataset = data.Subset(train_dataset, indices=[0,1] )
dataset['Title']=pd.Series([i.split(',')[1].split('.')[0].strip() for i in dataset.Name] )<feature_engineering>
train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2,
weight_index = 3)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )
dataset["Title"] = dataset["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset["Title"] = dataset["Title"].replace(['Ms', 'Mlle'], 'Miss')
 dataset["Title"] = dataset["Title"].replace(['Mme'], 'Mrs' )<feature_engineering>
fastText_wordEmbedder_f='.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
glove_wordEmbedder_f='.. /input/glove840b300dtxt/glove.840B.300d.txt'
glove_twitter = '.. /input/glove-global-vectors-for-word-representation/glove.twitter.27B.200d.txt'
Ticket=[]
for i in list(dataset['Ticket']):
if i.isdigit() :
Ticket.append('X')
 else:
Ticket.append(i.split(' ')[0])
 dataset['Ticket']=Ticket<feature_engineering>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float16')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path,dim = 300):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, dim))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def create_embedding_index(file,header=False):
f = open(file,'r')
 lines = []
if header:
line_vec = f.readlines() [1:]
else:
line_vec = f.readlines()
return dict(get_coefs(*line.strip().split(' ')) for line in line_vec )
Ticket = []
for i in list(dataset.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip())
 else:
Ticket.append(i)
 dataset['Ticket']=Ticket<categorify>
fastText_embedding_index,unknown_words_ft = \
build_matrix(tokenizer.word_index, fastText_wordEmbedder_f)
dataset = pd.get_dummies(dataset, columns = ["Cabin"], prefix="Cab")
 dataset = pd.get_dummies(dataset, columns = ["Embarked"], prefix="Em")
 dataset = pd.get_dummies(dataset, columns = ["Fsize"], prefix="Fam")
 dataset = pd.get_dummies(dataset, columns = ["Pclass"], prefix="Pc")
 dataset = pd.get_dummies(dataset, columns = ["Title"], prefix="Title")
 dataset = pd.get_dummies(dataset, columns = ["Ticket"], prefix="Tick" )<drop_column>
glove_embedding_index,_ = build_matrix(tokenizer.word_index,glove_wordEmbedder_f,300 )
dataset.drop(labels = ['Name','PassengerId'], axis = 1, inplace = True )<prepare_x_and_y>
glove_twitter,_ = build_matrix(tokenizer.word_index,glove_twitter,200 )
Y_train=dataset[:train_len]['Survived']
X_train=data[:train_len]
test=data[train_len:] 
<prepare_x_and_y>
embedding_matrix = np.concatenate([fastText_embedding_index, glove_embedding_index,glove_twitter], axis=-1)
 embedding_matrix.shape
del glove_embedding_index
del fastText_embedding_index
del glove_twitter
gc.collect()
<import_modules>
word_vec_size=300
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve 
<choose_model_class>
def get_embedding_matrix(word_index, embedding_index, word_vec_size):
toxinizer_vocab = len(word_index)+1
embedding_matrix = np.zeros(( toxinizer_vocab, word_vec_size))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return(embedding_matrix )
kfold = StratifiedKFold(n_splits=10 )<choose_model_class>
def seed_everything(seed=1234):
random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.backends.cudnn.deterministic = True
random_state = 42
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state = random_state))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","RandomForest","KNeighbours","LogisticRegression"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores")
<train_on_grid>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'], 'gamma': [ 0.001, 0.01, 0.1, 1],'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_<train_on_grid>
def weighted_avg(tensor,dim):
weight_array = np.arange(1,tensor.shape[dim] + 1)
 weight_array = weight_array / np.sum(weight_array)
 weight_array = torch.from_numpy(weight_array ).float().cuda()
for a in range(len(tensor.shape)) :
if a != dim:
weight_array = weight_array.unsqueeze(a)
 weighted_tensor = tensor * weight_array
return torch.sum(weighted_tensor,dim)
DTC = DecisionTreeClassifier()
dt_param_grid = {'max_features': ['auto', 'sqrt', 'log2'],'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
'min_samples_leaf':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],'random_state':[42]}
gsDTC = GridSearchCV(DTC,param_grid = dt_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsDTC.fit(X_train,Y_train)
 DTC_best = gsDTC.best_estimator_
gsDTC.best_score_<train_on_grid>
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, dropout_rate,
lstm_units, dense_hidden_units):
super(NeuralNet, self ).__init__()
vocab_size = embedding_matrix.shape[0]
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(vocab_size, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix,
dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(dropout_rate)
 self.lstm1 = nn.LSTM(embed_size,lstm_units,bidirectional=True,batch_first=True,num_layers=2)
 self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.dropout = nn.Dropout (.2)
 self.linear_out = nn.Linear(dense_hidden_units, 1)
 self.bn = nn.BatchNorm1d(dense_hidden_units)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm2, _ = self.lstm1(h_embedding)
 avg_pool_1 = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool_1), 1)
 h_conc_linear1 = F.relu(self.bn(self.linear1(h_conc)))
 h_conc_linear2 = self.dropout(F.relu(self.bn(self.linear2(h_conc))))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 return result
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],"max_features": [1, 3, 10],"min_samples_split": [2, 3, 10],"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],"n_estimators" :[100,300],"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,Y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_<train_on_grid>
def custom_loss(data, targets, weights):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=weights.float() )(data[:,:1].float() ,targets[:,:1].float())
 return bce_loss_1
LRC = LogisticRegression()
lr_param_grid = {'penalty':['l1', 'l2'],'C': np.logspace(0, 4, 10)}
gsLRC=GridSearchCV(LRC,param_grid=lr_param_grid,cv=kfold,scoring='accuracy', n_jobs= 4, verbose = 1)
 gsLRC.fit(X_train,Y_train)
 LRC_best = gsLRC.best_estimator_
gsLRC.best_score_<train_on_grid>
def sigmoid(x):
return 1/(1+np.exp(-x))
KNNC = KNeighborsClassifier()
knn_param_grid = {'n_neighbors':[3, 4, 5, 6, 7, 8],'leaf_size':[1, 2, 3, 5],
'weights':['uniform', 'distance'],'algorithm':['auto', 'ball_tree','kd_tree','brute']}
gsKNNC=GridSearchCV(KNNC,param_grid=knn_param_grid,cv=kfold,scoring='accuracy',n_jobs=4,verbose=1)
 gsKNNC.fit(X_train,Y_train)
 KNN_best=gsKNNC.best_estimator_
gsKNNC.best_score_<train_model>
def train_model(learn,test,output_dim,lr=0.002,
batch_size=512, n_epochs=5,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))).schedule_hp('wd',1e-2)for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(3)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
votingC = VotingClassifier(estimators=[('svc', SVMC_best),('rfc', RFC_best),('lrc', LRC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, Y_train )<save_to_csv>
print('fastText Model')
 seed_everything(1234 + 0)
 fastText_model = NeuralNet(embedding_matrix,dropout_rate=DROPOUT_RATE,lstm_units=LSTM_UNITS,dense_hidden_units=DENSE_HIDDEN_UNITS)
 fastText_learn = Learner(databunch, fastText_model, loss_func=custom_loss)
 fastText_test_preds = train_model(fastText_learn,test_dataset,output_dim=1,batch_size = batch_size,n_epochs = EPOCHS)
 fastText_model_submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': np.squeeze(fastText_test_preds[:,0])
 })
 fastText_model_submission.to_csv('submission.csv', index=False)
<save_to_csv>
test_Survived = votingC.predict(test ).astype(int)
 submission = pd.DataFrame({
"PassengerId": IDtest,
"Survived": test_Survived
})
 submission.to_csv('Titanic_test_prediction_V9.csv', index=False )<compute_test_metric>