Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<save_to_csv>
use_own_text = True
use_train_data = False
output = pd.DataFrame({'PassengerId' : test_data.PassengerId, 'Survived' : prediction})
 output.to_csv('my_submissions.csv', index = False )<set_options>
BERT_MODEL_PATHS = [
'bert_bert_sb_0.93405_out_1_cpa.pt',
'bert_wss_boosted2_cpa_nlabels31_0.93774.pt',
'bert_wss_aux_ot_rt_0.93803.pt',
'bert_sb1t9_no_identity_ot_rt_cont_rand_sb1t9_0.93574_out_13_cpa.pt',
'bert_base_cont_boosted3_sge6_0.93672_out_12_cpa.pt',
'bert_base_last_year_competition_cont_pp_no_identity_cont__0.94643_out_1_cpa.pt',
'bert_sb_mini_epoch_0.93451_out_8_cpa.pt',
'bert_wss_aux_ot_0.93722.pt',
'bert_public_0.93373.pt',
'bert_subgroup_balanced_20p_target_ist_bert_ga2_0.93370.pt',
'bert_yuval_baseline_bert_model_cont_boost_0.93352.pt',
'bert_base_0.93327_out_1_cpa.pt',
]
BERT_MODEL_PATHS = [DATASET_PATH+bmp for bmp in BERT_MODEL_PATHS]
warnings.filterwarnings(action="ignore")
 %pylab inline
plt.style.use('seaborn-poster')
 sns.set(font_scale=2)
 random_seed = 123456789
np.random.seed(random_seed)
 mean_squared_error,mean_squared_log_error,r2_score,make_scorer
def getMetrics(val_reels, val_predites):
correlations = pd.DataFrame(list(zip(val_predites,val_reels)) , columns=['predictions','valeurs'])
 return sqrt(mean_squared_error(val_reels, val_predites)) ,sqrt(abs(r2_score(val_reels, val_predites))),correlations.astype(float ).corr().valeurs[0]
def affichageMetricsSansLog(val_reels, val_predites):
print("Mean absolute error MAE : %.6f" % mean_absolute_error(val_reels, val_predites))
print("Mean squared error MSE : %.6f" % mean_squared_error(val_reels, val_predites))
print("Root mean squared error RMSE : %.6f" % sqrt(mean_squared_error(val_reels, val_predites)))
 print("-"*50,"
\tPour comparer voici
\tla distribution des valeurs de la variable :
","-"*50,"
",pd.Series(val_reels ).describe() ,"
","-"*50,)
 correlations = pd.DataFrame(list(zip(val_predites,val_reels)) , columns=['predictions','valeurs'])
 sns.set(font_scale=2)
 plt.figure(figsize=(8,8))
plt.title('Correlation Pearson des variables', y=1.05, size=24)
 ax = sns.heatmap(correlations.astype(float ).corr() ,linewidths=0.3,vmax=1.0,fmt ="1.4f",
square=True, cmap='coolwarm', linecolor='white', annot=True)
 ax.set_xlim(0,2);
ax.set_ylim(2,0);
plt.show()
sns.set(font_scale=2)
 sns.jointplot(x='valeurs',y='predictions',
data=correlations, kind='reg', size=25)
 noms = [
"LinearRegressionRidge",
"BayesianRidge",
"SVR_linear",
]
regresseurs = [
Ridge() ,
BayesianRidge() ,
SVR(kernel='linear', C=0.005),
]
regresseurs = { nom:reg for nom,reg in zip(noms,regresseurs)}
modelRFE = {}
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})".format(
 results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("")
 def choixModelsRegressions(regresseurs):

 noms = [x for x in regresseurs]
print('Initialisation des models
---Si vous sesisez un ou plusieurs caratères le modele est effacé !---
')
 for model in noms:
if input('Model : '+model+' -- ')!= '':
regresseurs.pop(model)
 def calculRegressions(X_train, X_test, y_train, y_test):
rmse,r,p = dict() ,dict() ,dict()
yRegressions = pd.DataFrame()
yRegressions['Observations']=y_test
for nom in regresseurs:
t1 = time.time()
yRegressions[nom] = regresseurs[nom].fit(X_train, y_train ).predict(X_test)
 rmse[nom],r[nom],p[nom] = getMetrics(y_test, yRegressions[nom])
 print('Exécution '+nom.ljust(30)+(': %.2fs' %(time.time() - t1)).lstrip('0')+(' p = %.12f' % p[nom] ).lstrip('0'))
resultats = pd.DataFrame(pd.Series(rmse),columns=["RMSE"])
 resultats["R"] = pd.Series(r)
 resultats["Pearson"] = pd.Series(p)
 resultats.sort_values(by='Pearson',ascending=False, inplace=True)
 return resultats,yRegressions, regresseurs
def initRFE(X, y, regresseurs, modelRFE):
for nom in regresseurs:
t1 = time.time()
modelRFE[nom] = RFE(regresseurs[nom])
 modelRFE[nom].fit(X,y)
 return modelRFE
def calculRegressionsRFE(X, y, regresseurs, modelRFE, modelOcSVM=None, taux = 0.3):
rmse,r,p = dict() ,dict() ,dict()
for nom in regresseurs:
t1 = time.time()
X_train, X_test, y_train, y_test = train_test_split(
 X,
y,
test_size=taux,
random_state=random_seed 
)
 if modelOcSVM is not None :
newX = modelOcSVM.predict(X_train)
 X_train, y_train = X_train[newX != -1], y_train[newX != -1]
else :
newX = np.ones(X_train.shape[0])
 X_train, X_test = X_train[X_train.columns[modelRFE[nom].support_]],X_test[X_test.columns[modelRFE[nom].support_]]
yRegressions[nom] = regresseurs[nom].fit(X_train, y_train ).predict(X_test)
 rmse[nom],r[nom],p[nom] = getMetrics(y_test, yRegressions[nom])
 print('Exécution '+nom.ljust(30)+(': %.2fs' %(time.time() - t1)).lstrip('0')+(' p = %.12f' % p[nom] ).lstrip('0'))
resultats = pd.DataFrame(pd.Series(rmse),columns=["RMSE"])
 resultats["R"] = pd.Series(r)
 resultats["Pearson"] = pd.Series(p)
 resultats.sort_values(by='Pearson',ascending=False, inplace=True)
 return resultats,yRegressions,regresseurs,modelRFE<load_from_csv>
BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_sb2t8_0.91725.pt')
 BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_aux_wss_bad_minor_0.91675.pt')
 BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_aux_wss_0.92162.pt' )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 donnees = pd.concat([test,train],sort=False)
 print('Train size : ',train.shape,'
Test size :',test.shape)
 donnees.tail()<drop_column>
package_dir = '.. /input/pytorch-pretrained-bert-master/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master/'
sys.path.append(package_dir )
Cible = 'SalePrice'
colQualitatives = ['MSSubClass', 'OverallQual', 'OverallCond'] + list(donnees.dtypes[donnees.dtypes == 'object'].reset_index() ['index'].values)
 colQuantitatives = list(donnees.drop(columns=colQualitatives + [Cible] ).columns )<count_missing_values>
args = {
'device' : 'cuda',
'learning_rate' : 25e-6,
'warmup_proportion': 0.2,
'batch_size' : 32,
'ebbs_epoch_samples' : 25e3,
'num_eebs_epochs' : 15e6//100e3,
'ebbs_test_multipl' : 1,
'eval_sample_size' : 10e3,
'num_labels' : 1,
'max_seq_length' : 250,
'gradient_accumulation_steps' : 2,
'fp16' : False,
'seed' : 70135
}
donnees[colQuantitatives].isna().sum() [
donnees[colQuantitatives].isna().sum() > 0
].sort_values()<count_missing_values>
label_clms = ['target','severe_toxicity', 'obscene','identity_attack',
'insult', 'threat','funny', 'wow','sad','sexual_explicit','rating']
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
target_clms = [*label_clms,*identity_columns]
variables = donnees[colQuantitatives].isna().sum() [ 
(donnees[colQuantitatives].isna().sum() > 0)& 
(donnees[colQuantitatives].isna().sum() < 24)
 ].reset_index() ['index']
for variable in variables:
donnees[variable].fillna(train[variable].mean() ,inplace=True )<count_missing_values>
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )
donnees[colQuantitatives].isna().sum() [
donnees[colQuantitatives].isna().sum() > 0
].sort_values()<predict_on_test>
def replace_underscore_with_backspace(txt):

 return re.sub('_+',' ',txt)
 def replace_newline_with_dot(txt):

 return re.sub('
+','.',txt)
 def replace_newline_with_space(txt):

 return re.sub('
+',' ',txt)
 def remove_urls(txt):

 return re.sub(r"(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+
def remove_multi_symbols(txt,symbols=string.punctuation):

 for symbol in symbols:
txt = re.sub('\'+symbol+'+',symbol,txt)
 return txt
y_pred = np.exp(modelRL.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))<predict_on_test>
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 if use_train_data:
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv" ).sample(10000 )
regresseur = make_pipeline(PolynomialFeatures(4), Ridge())
 regresseur.fit(modelSS.fit_transform(X),np.log(y))
y_pred = np.exp(regresseur.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))<predict_on_test>
if use_own_text:
my_text = [
'asdf',
'feck-you',
'trolling the nation',
'I am proud to be gay',
'Nothing to do here',
'Hey, keep your Islamophobia out of public domains.',
'Well, the money the education system spent on you was wasted',
'This clown confuses the word "substance" with "abuse".',
'"Two things are infinite: the universe and human stupidity; and I´m not sure about the universe." - Albert Einstein',
'I have epilepsy',
'I am a woman with epilepsy',
'I am a person who uses a wheelchair',
'I use a wheelchair',
'I am gay and have to use a wheelchair',
'I am blind',
'I am a person who is deaf',
'I am a woman who is deaf',
'I am a non-binary-gender person who is deaf',
'I am a black person who is deaf',
'I am pround to be gay',
'I am a proud homo',
'I am a proud lesbian',
'I am a proud to be black',
'KKK',
'a55hle',
'a$$hole',
'I am a very proud lesbian',
'I am a blind homosexual.',
"I don't feel ashamed for beeing lesbian",
"Proud to be a fu**ing gay person",
"Totally wrong to think that this wouldn't be gay as shit",
' ',
'!',
'',
'???',
'I hate trump',
'I hate obama',
'Trump',
'I like that you are gay',
]
test_df = pd.DataFrame({
'id': range(len(my_text)) ,
'comment_text': my_text
} )
X = donnees[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)][dataGarageYrBlt[1:]]
y = donnees[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)].GarageYrBlt
y_pred = pd.Series(np.exp(regresseur.predict(modelSS.fit_transform(X))), index=y.index)
 donnees.GarageYrBlt[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)] = y_pred
donnees.GarageYrBlt = donnees.GarageYrBlt.astype('int16' )<prepare_x_and_y>
test_df['comment_text_orig'] = test_df['comment_text']
X = donnees[(~donnees.LotFrontage.isna())&(donnees.LotFrontage < 175)][dataLotFrontage[1:]]
y = donnees[(~donnees.LotFrontage.isna())&(donnees.LotFrontage < 175)].LotFrontage.ravel()
modelSS = StandardScaler()
modelRL = LinearRegression()
modelRL.fit(modelSS.fit_transform(X),np.log(y))
coef = pd.DataFrame({'Coef':modelRL.coef_})
 coef['Label'] = X.columns
coef.sort_values(by='Coef' )<predict_on_test>
functions = [
remove_urls,
replace_underscore_with_backspace,
replace_newline_with_dot,
lambda k: remove_multi_symbols(k,symbols=['.','-','='])
 ]
for func in functions:
print(func.__name__)
 test_df['comment_text'] = test_df['comment_text'].apply(func )
y_pred = np.exp(modelRL.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))<predict_on_test>
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = np.zeros(( len(example),max_seq_length+2))
token_len = np.zeros(len(example))
for i,text in enumerate(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
tokens = tokenizer.convert_tokens_to_ids(tokens_a)
 tokens = [101, *tokens, 102]
token_len[i] = len(tokens)
 all_tokens[i,:len(tokens)] = np.asarray(tokens)
 return all_tokens, token_len
regresseur = make_pipeline(PolynomialFeatures(4), Ridge())
 regresseur.fit(modelSS.fit_transform(X),np.log(y))
y_pred = np.exp(regresseur.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))<predict_on_test>
SEED = 1234
TOKENIZER_PATH = '.. /input/berttokenizer/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
tokenizer = BertTokenizer.from_pretrained(TOKENIZER_PATH, cache_dir=None,do_lower_case=True )
X = donnees[donnees.LotFrontage.isna() ][dataLotFrontage[1:]]
y = donnees[donnees.LotFrontage.isna() ].LotFrontage
y_pred = pd.Series(np.exp(regresseur.predict(modelSS.fit_transform(X))), index=y.index)
 donnees.LotFrontage[ donnees.LotFrontage.isna() ] = y_pred
donnees.LotFrontage = donnees.LotFrontage.astype('int16' )<count_missing_values>
test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test, token_len = convert_lines(test_df["comment_text"], args['max_seq_length'], tokenizer )
donnees[colQualitatives].isna().sum() [
donnees[colQualitatives].isna().sum() > 0
].sort_values()<count_missing_values>
test_df['token_len'] = token_len
for col in donnees[colQualitatives].isna().sum() [
donnees[colQualitatives].isna().sum() > 0
].reset_index() ['index']:
print("%12s"%col,'-',"%2d"%len(donnees[col].unique()),' - ',donnees[col].unique().tolist() [:8])
 donnees[colQualitatives] = donnees[colQualitatives].fillna('Non Affectée' )<data_type_conversions>
device = torch.device('cuda')
 class BertForMultiLabelSequenceClassification(BertForSequenceClassification):

 def __init__(self, config, num_labels=2):
super(BertForMultiLabelSequenceClassification,
self ).__init__(config, num_labels)
 self.num_labels = num_labels
self.bert = BertModel(config)
 self.dropout = torch.nn.Dropout(config.hidden_dropout_prob)
 self.classifier = torch.nn.Linear(config.hidden_size, num_labels)
 self.apply(self.init_bert_weights)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, return_pooled_output=False, return_last_unpooled_layer=False):
full_output, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 logits = self.classifier(pooled_output)
 if labels is not None:
loss_fct = BCEWithLogitsLoss()
mask =(labels != -1 ).to(dtype=torch.float)
 logits = logits * mask
label_tensor = labels * mask
loss = loss_fct(logits, labels)
 return loss, logits
elif return_pooled_output: return logits, pooled_output
elif return_last_unpooled_layer: return logits, full_output
return logits
def forward_old(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, loss_weight=None):
_, pooled_output = self.bert(
 input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 if labels is not None:
loss_fct = BCEWithLogitsLoss()
mask =(labels != -1 ).to(dtype=torch.float)
 logits = logits * mask
labels = labels * mask
loss = loss_fct(logits.view(-1, self.num_labels),
labels.view(-1, self.num_labels))
return loss
else:
return logits
def freeze_bert_encoder(self):
for param in self.bert.parameters() :
param.requires_grad = False
def unfreeze_bert_encoder(self):
for param in self.bert.parameters() :
param.requires_grad = True
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class BertLSTM1(nn.Module):

 def __init__(self, embedding_matrix, num_labels, n_aux_units = 7, max_features=args['max_seq_length'],LSTM_UNITS=128,DENSE_HIDDEN_UNITS=4*128):
super(BertLSTM2, self ).__init__()
self.num_labels = num_labels
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, num_labels)
 def forward(self, input_ids, attention_mask=None, labels=None, loss_weight=None, return_pooled_output=False):
h_embedding = self.embedding(input_ids)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 return result
donnees.BsmtFullBath = donnees.BsmtFullBath.round(2 ).astype('str')
 colQualitatives.append('BsmtFullBath')
 donnees.BsmtHalfBath = donnees.BsmtHalfBath.round(2 ).astype('str')
 colQualitatives.append('BsmtHalfBath')
 donnees.GarageCars = donnees.GarageCars.round(2 ).astype('str')
 colQualitatives.append('GarageCars' )<drop_column>
def load_bert_model(path, num_labels=args['num_labels']):
state_dict = torch.load(path)
 if path.find('.. /input/bertlstm-weights/')==0:
print('loading bertLSTM model')
 if num_labels is None:
num_labels = int(state_dict[list(state_dict.keys())[-1]].shape[0])
 print('Number of Labels set to',num_labels)
 try:
model = BertLSTM1(np.zeros(( 30522, 768)) ,num_labels)
 print(model.load_state_dict(state_dict))
except:
print('trying BertLSTM2')
 model = BertLSTM2(np.zeros(( 30522, 768)) ,num_labels)
 print(model.load_state_dict(state_dict))
model = model.to('cuda')
 del state_dict
return model
config = BertConfig(30522)
 if num_labels is None:
if 'classifier.bias' in state_dict.keys() :
num_labels = int(state_dict['classifier.bias'].shape[0])
 else:
print('No classifier.bias in state_dict --> assuming num_labels = 1')
 num_labels = 1
model = BertForMultiLabelSequenceClassification(config,num_labels=num_labels)
 model.load_state_dict(state_dict)
 model.to(device)
 del state_dict
return model
donnees.drop(columns=['MoSold','YrSold'],inplace=True )<count_unique_values>
out_clms = []
for ii, BERT_MODEL_PATH in enumerate(BERT_MODEL_PATHS):
print('
using model',BERT_MODEL_PATH)
 model = load_bert_model(BERT_MODEL_PATH,num_labels=None)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
buckets = 30
old_thld = -1
for b in range(buckets):
thld = int(np.percentile(test_df.token_len,(b+1)*100//buckets))
mask =(test_df.token_len>old_thld)&(test_df.token_len<=thld)
 if mask.sum() >0:
batch_size = args['batch_size'] if thld<=220 else 16
old_thld = thld
print('\rBucketNr',b,'thld:',thld,'samples:',mask.sum() ,end='')
 preds = None
test = torch.utils.data.TensorDataset(torch.tensor(X_test[mask,:thld], dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(x_batch,)in enumerate(test_loader):
logits = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 y_pred = logits.sigmoid().detach().cpu().numpy()
if preds is None: preds = y_pred
else: preds = np.concatenate(( preds,y_pred),axis=0)
 test_df.loc[mask,'bo'+str(ii)] = preds[:,0]
out_clms.append('bo'+str(ii))
for col in ['FullBath','HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd','Fireplaces','MiscVal']:
colQualitatives.append(col)
 print("%13s"%col,'-',"%3d"%len(donnees[col].unique()),' - ',
donnees[col].sort_values().unique().tolist() [:15] )<data_type_conversions>
weights = np.ones(len(out_clms))
print([(BERT_MODEL_PATHS[i].split('/')[-1],weights[i])for i in range(len(weights)) ])
 test_df.fillna(0,inplace=True)
 test_df['prediction'] = np.average(test_df[out_clms].values, weights=weights, axis=1 )
for col in donnees.dtypes[donnees.dtypes == 'category'].reset_index() ['index'] :
donnees[col] = donnees[col].astype('str' )<create_dataframe>
use_ranked_sum = False
if use_ranked_sum:
for key in out_clms:
test_df[key + '_rank'] = test_df[key].rank()
test_df['rank_sum'] = np.sum(
 test_df[col] for col in test_df.columns if '_rank' in col)
 test_df['rank_sum_prediction'] = test_df['rank_sum']/(len(out_clms)*
test_df.shape[0] )
modelVect = DictVectorizer()
modelVect.fit(donnees[colQualitatives].to_dict('records'))
donneesQualitatives = modelVect.transform(donnees[colQualitatives].to_dict('records'))
donneesQualitatives = pd.DataFrame(
 donneesQualitatives.toarray() ,
columns=modelVect.feature_names_,
index=donnees.index
)<normalization>
if use_train_data:
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
target_columns = ['severe_toxicity', 'obscene','identity_attack', 'insult', 'threat']
offtarget_columns = ['funny', 'wow','sad', 'likes', 'disagree', 'sexual_explicit']
TOXICITY_COLUMN = 'target'
SUBGROUP_AUC = 'subgroup_auc'
BPSN_AUC = 'bpsn_auc'
BNSP_AUC = 'bnsp_auc'
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def compute_subgroup_auc(df, subgroup, label, model_name):
subgroup_examples = df[df[subgroup]]
return compute_auc(subgroup_examples[label], subgroup_examples[model_name])
 def compute_bpsn_auc(df, subgroup, label, model_name):

 subgroup_negative_examples = df[df[subgroup] & ~df[label]]
non_subgroup_positive_examples = df[~df[subgroup] & df[label]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bnsp_auc(df, subgroup, label, model_name):

 subgroup_positive_examples = df[df[subgroup] & df[label]]
non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bias_metrics_for_model(dataset,subgroups,model,label_col,include_asegs=False):

 dataset = convert_dataframe_to_bool(dataset)
 records = []
for subgroup in subgroups:
record = {
'subgroup': subgroup,
'subgroup_size': len(dataset[dataset[subgroup]])
 }
record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)
 record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)
 record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)
 records.append(record)
 return pd.DataFrame(records ).sort_values('subgroup_auc', ascending=True)
 def calculate_overall_auc(df, model_name, TOXICITY_COLUMN):
true_labels = df[TOXICITY_COLUMN]>=0.5
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC], POWER),
power_mean(bias_df[BPSN_AUC], POWER),
power_mean(bias_df[BNSP_AUC], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score), bias_score
def get_scores(in_df, model_name,TOXICITY_COLUMN='target',fillna=True,consolidation_method='mean'):
df = in_df[[model_name,'id',*identity_columns, TOXICITY_COLUMN]].groupby('id')
 if consolidation_method=='mean': df = df.mean()
if consolidation_method=='max': df = df.max()
if consolidation_method=='min': df = df.min()
bias_metrics_df = compute_bias_metrics_for_model(df, identity_columns, model_name, TOXICITY_COLUMN)
 if fillna: bias_metrics_df.fillna(0.5,inplace=True)
 try:
overall_auc = calculate_overall_auc(df, model_name, TOXICITY_COLUMN)
 except ValueError:
print('Error - only one class present in df')
 overall_auc = np.nan
final_metric, bias_score = get_final_metric(bias_metrics_df, overall_auc)
 bias_metrics_df['overall_auc']=overall_auc
bias_metrics_df['final_bias_score'] = bias_score
bias_metrics_df['final_metric'] = final_metric
bias_metrics_df.index = bias_metrics_df.subgroup
return bias_metrics_df
colQuantitatives = list(donnees.drop(columns=colQualitatives + [Cible] ).columns)
 modelSS = StandardScaler()
donneesQuantitatives = pd.DataFrame(
 modelSS.fit_transform(donnees[colQuantitatives]),
columns=donnees[colQuantitatives].columns,
index=donnees.index 
)
 donneesQuantitatives.head()<create_dataframe>
if use_train_data:
score = get_scores(test_df,'prediction' )
yeoJohn = PowerTransformer(method='yeo-johnson')
 donneesQuantitatives = pd.DataFrame(
 yeoJohn.fit_transform(donneesQuantitatives),
columns=donneesQuantitatives.columns,
index=donneesQuantitatives.index
)<merge>
if use_train_data:
for i,clm in enumerate(out_clms):
print(i, str(get_scores(test_df,clm ).final_metric.mean())[:5],weights[i],BERT_MODEL_PATHS[i].split('/')[-1],sep='\t' )
donneesFinal = donneesQuantitatives.join(donneesQualitatives)
 donneesFinal['SalePrice'] = donnees.SalePrice
donneesTrain = donneesFinal[~donneesFinal.SalePrice.isna() ]
donneesTest = donneesFinal[donneesFinal.SalePrice.isna() ]
donneesTest.drop(columns='SalePrice', inplace=True)
 donneesTrain.shape,donneesTest.shape<split>
for out_clm in ['prediction',*out_clms]:
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_df[out_clm]
})
 if out_clm=='prediction': out_clm = 'blend'
submission.to_csv('submission_'+out_clm+'.csv', index=False )
X_train, X_test, y_train, y_test = train_test_split(
 donneesTrain.drop(columns='SalePrice' ).values,
np.log(donneesTrain['SalePrice'].values),
test_size=0.3,
random_state=random_seed 
)
 X_train.shape, X_test.shape<compute_test_metric>
test_df.to_csv('test_df.csv', index=False )
resultats,yRegressions,regresseurs = calculRegressions(X_train, X_test, y_train, y_test)
 resultats<predict_on_test>
pd.concat(( test_df['comment_text'],(test_df[[*out_clms,'prediction']]*100 ).astype(np.int)) ,axis=1 )