Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<load_from_csv>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
df = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 feat = totally_cleaned(feat)
 feat = feature_engineering(feat)
 feat = catboost_encode(feat,['Title','Cabin_Location'])
 feat = bin_data(feat,['Age'],5,False)
 feat = bin_data(feat,['Fare'],10,True)
 feat = onehot_encode(feat,['Embarked'])
 feat = get_dummy(feat,['Ticket_Pre'])
 feat = drop_catagotical(feat)
 dtrain = feat[feat.Survived.notnull() ].copy()
dtest = feat[feat.Survived.isnull() ].copy()
dtest = dtest.drop(['Survived'], axis=1)
 dtest.index = df_test.index
print(f"Raw data shape : {df.shape} {df_test.shape}")
 print(f"Clean data shape : {dtrain.shape} {dtest.shape}" )<compute_train_metric>
identities = ['male','female','transgender','other_gender','heterosexual','homosexual_gay_or_lesbian',
'bisexual','other_sexual_orientation','christian','jewish','muslim','hindu','buddhist',
'atheist','other_religion','black','white','asian','latino','other_race_or_ethnicity',
'physical_disability','intellectual_or_learning_disability','psychiatric_or_mental_illness',
'other_disability']
train_labeled_df = train.loc[:, ['target'] + identities ].dropna()
toxic_df = train_labeled_df[train_labeled_df['target'] >=.5][identities]
non_toxic_df = train_labeled_df[train_labeled_df['target'] <.5][identities]
toxic_count = toxic_df.where(train_labeled_df == 0, other = 1 ).sum()
non_toxic_count = non_toxic_df.where(train_labeled_df == 0, other = 1 ).sum()
toxic_vs_non_toxic = pd.concat([toxic_count, non_toxic_count], axis=1)
 toxic_vs_non_toxic = toxic_vs_non_toxic.rename(index=str, columns={1: "non-toxic", 0: "toxic"})
 toxic_vs_non_toxic.sort_values(by='toxic' ).plot(kind='bar', stacked=True, figsize=(30,10), fontsize=20 ).legend(prop={'size': 20} )
X = dtrain[dtrain['Fare']<300].drop(["Survived"], axis=1)
 y = dtrain[dtrain['Fare']<300].Survived
metric = 'accuracy'
rf = RandomForestClassifier()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{cross_val_score(rf, X, y, cv=kfold, scoring=metric ).mean() *100:.4f} % Accuracy")
<choose_model_class>
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType
metric = 'accuracy'
kfold = KFold(n_splits=10, shuffle=True)
 random_state = 1
classifiers = [
GaussianProcessClassifier(random_state = random_state),
Perceptron(random_state = random_state),
RidgeClassifier(random_state = random_state),
SGDClassifier(random_state = random_state),
SVC(random_state = random_state),
RandomForestClassifier(random_state = random_state),
GradientBoostingClassifier(random_state = random_state),
lgb.LGBMClassifier(random_state = random_state),
xgb.XGBClassifier(objective="reg:squarederror",random_state = random_state),
xgb.XGBClassifier(objective="reg:squarederror",booster='dart',random_state = random_state),
AdaBoostClassifier(random_state = random_state),
BaggingClassifier(random_state = random_state),
KNeighborsClassifier() ,
LogisticRegression(random_state = random_state)
 ]
clf_names = [
"GaussianProcessClassifier",
"Perceptron",
"Ridge",
"SGDClassifier",
"SVC",
"rndmforest",
"gbmC",
"lgbm",
"xgboost",
"xgboost_dart",
"AdaBoost",
"Bagging",
"KNeighbors",
"Logistic"
]
cv_results = []
for clf in classifiers:
cv_results.append(cross_val_score(clf, X, y, cv=kfold, scoring=metric,n_jobs=4)) 
<train_on_grid>
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
ridge = RidgeClassifier()
ridge_pg = {'alpha':[1,10,100],'tol':[0.001,0.0001,0.00001]}
gs_ridge = GridSearchCV(ridge,param_grid = ridge_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gs_ridge.fit(X,y)
 ridge_best = gs_ridge.best_estimator_<train_on_grid>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
lrc = LogisticRegression()
lrc_pg = {'penalty':['l1','l2','elasticnet'],'max_iter':[100,500,1000],'warm_start':[True],'tol':[0.001,0.0001,0.00001],'solver':['saga']
,'l1_ratio':[0.5]}
gs_lrc = GridSearchCV(lrc,param_grid = lrc_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gs_lrc.fit(X,y)
 lrc_best = gs_lrc.best_estimator_<train_on_grid>
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
RFC = RandomForestClassifier()
rf_pg = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X,y)
 RFC_best = gsRFC.best_estimator_<train_on_grid>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
XGBC =xgb.XGBClassifier(objective="reg:squarederror",random_state = random_state)
 if(debug):
XGBC_pg = {'max_depth':[2]}
else:
XGBC_pg = {'max_depth':[2,4,100],'learning_rate':[0.0001,0.001,0.005],'n_estimators':[100,250,500,1000],
'reg_alpha':[0.00001,0.00005,0.0001,0.0005],'colsample_bytree':[.5,.6,.7,.8]}
gsXGBC = GridSearchCV(XGBC,param_grid = XGBC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsXGBC.fit(X,y)
 XGBC_best = gsXGBC.best_estimator_<train_on_grid>
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
XGBDC =xgb.XGBClassifier(objective="reg:squarederror",booster = 'dart',random_state = random_state)
 if(debug):
XGBDC_pg = {'max_depth':[2]}
else:
XGBDC_pg = {'max_depth':[2,4],'learning_rate':[0.0001,0.001,0.005],'n_estimators':[100,250,500],
'reg_alpha':[0.00001,0.00005,0.0001,0.0005],'colsample_bytree':[.5,.7,.8]}
gsXGBDC = GridSearchCV(XGBDC,param_grid = XGBDC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsXGBDC.fit(X,y)
 XGBDC_best = gsXGBDC.best_estimator_<train_on_grid>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
BaggC = BaggingClassifier()
BaggC_pg = {'n_estimators':[5,10,25],'bootstrap':[False,True],'max_features':[.1,.4,.5,.7],
'max_samples':[.2,.5,.7,1]}
gsBaggC = GridSearchCV(BaggC,param_grid = BaggC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsBaggC.fit(X,y)
 BaggC_best = gsBaggC.best_estimator_<train_on_grid>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
adac = AdaBoostClassifier()
if(debug):
adac_pg = {'algorithm':['SAMME']}
else:
adac_pg = {'algorithm':['SAMME','SAMME.R'],'learning_rate':[.001,.00001,.01,.5,1],'n_estimators':[25,50,100,150,200]}
gsADAC = GridSearchCV(adac,param_grid = adac_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsADAC.fit(X,y)
 ADAC_best = gsADAC.best_estimator_<feature_engineering>
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 contraction_mapping = {"Trump's": "Trump is","ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def known_contractions(text):
known = []
for contract in contraction_mapping:
if contract in text:
known.append(contract)
 return known
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
data = data.astype(str ).apply(lambda x: replace_typical_misspell(x))
data = data.astype(str ).apply(lambda x: clean_numbers(x))
data = data.astype(str ).apply(lambda x: known_contractions(x))
return data
def important_features(names_classifiers):
ncols = 2
nrows = int(np.ceil(len(names_classifiers)/ncols))
nclassifier = 0
for row in range(nrows):
for col in range(ncols):
if(nclassifier == len(names_classifiers)) :
break
name = names_classifiers[nclassifier][0]
classifier = names_classifiers[nclassifier][1]
indices = np.argsort(classifier.feature_importances_)[::-1][:10]
g = sns.barplot(y=X.columns[indices][:10],x =(classifier.feature_importances_/(classifier.feature_importances_.sum())) [indices][:10] , orient='h',ax=axes[row][col])
 g.set_xlabel("Relative importance",fontsize=10)
 g.set_ylabel("Features",fontsize=10)
 g.tick_params(labelsize=8)
 g.set_title(name + " feature importance")
 nclassifier += 1<predict_on_test>
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
test_RFC = pd.Series(RFC_best.predict(dtest), name="RFC")
 test_lrc = pd.Series(lrc_best.predict(dtest), name="LRC")
 test_XGBC = pd.Series(XGBC_best.predict(dtest), name="XGBoost")
 test_XGBDC = pd.Series(XGBDC_best.predict(dtest), name="XGBoost_Dart")
 test_BaggC = pd.Series(BaggC_best.predict(dtest), name="Bagging")
 test_Ridge = pd.Series(ridge_best.predict(dtest), name="Ridge")
 test_ADAC = pd.Series(ADAC_best.predict(dtest), name="ADAC")
 ensemble_results = pd.concat([test_RFC,test_lrc,test_XGBC,test_XGBDC, test_BaggC,test_Ridge,test_ADAC],axis=1)
 train_RFC = pd.Series(RFC_best.predict(X), name="RFC")
 train_lrc = pd.Series(lrc_best.predict(X), name="LRC")
 train_XGBC = pd.Series(XGBC_best.predict(X), name="XGBoost")
 train_XGBDC = pd.Series(XGBDC_best.predict(X), name="XGBoost_Dart")
 train_BaggC = pd.Series(BaggC_best.predict(X), name="Bagging")
 train_Ridge = pd.Series(ridge_best.predict(X), name="Ridge")
 train_ADAC = pd.Series(ADAC_best.predict(X), name="ADAC")
 ensemble_results_train = pd.concat([train_RFC,train_lrc,train_XGBC,train_XGBDC, train_BaggC,train_Ridge,train_ADAC],axis=1)
 g= sns.heatmap(ensemble_results.corr() ,annot=True)
<train_on_grid>
x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
lgbmc = lgb.LGBMClassifier()
if(debug):
lgbmc_pg = {'num_leaves':[30]}
else:
lgbmc_pg = {'num_leaves':[30,50,60],'max_depth':[2,3,7,-1],'learning_rate':[.001,.00001,.01,.5],'n_estimators':[100,150,200,500]}
gsLGBMC = GridSearchCV(lgbmc,param_grid = lgbmc_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 ensemble_results.head(3)
 gsLGBMC.fit(ensemble_results_train,y)
 LGBMC_best = gsLGBMC.best_estimator_
g = plot_learning_curve(LGBMC_best,"Emsemble LGBM learning curves",ensemble_results_train,y,cv=kfold)
<save_to_csv>
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
test_Survived = pd.Series(LGBMC_best.predict(ensemble_results), name="Survived")
 output = pd.DataFrame({'PassengerId': dtest.index,
'Survived': test_Survived.astype('int32')})
 output.to_csv("ensemble_python_voting.csv",index=False )<set_options>
max_features = None
%matplotlib inline
%matplotlib inline
rcParams['figure.figsize'] = 12, 4
warnings.filterwarnings('ignore' )<load_from_csv>
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
data_raw = pd.read_csv('.. /input/titanic/train.csv')
 data_val = pd.read_csv('.. /input/titanic/test.csv')
 data_train = data_raw.copy(deep = True)
 data_test = data_val.copy(deep = True)
 print(data_train.info())
 print("
print(data_test.info())
 print("

 data_combine = [data_train, data_test] 
<sort_values>
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
data_train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
data_train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
data_train[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
data_train[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )
data_train[["Embarked", "Survived"]].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
x_test_torch = torch.tensor(x_test, dtype=torch.long )
for dataset in data_combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(data_train['Title'], data_train['Sex'] )<feature_engineering>
batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )
for dataset in data_combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 data_train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in data_combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 data_train.head()<drop_column>
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )
data_train = data_train.drop(['Name', 'PassengerId'], axis=1)
 data_test = data_test.drop(['Name'], axis=1)
 data_combine = [data_train, data_test]
data_train.shape, data_test.shape<categorify>
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )