Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
tqdm.pandas()
df_all.loc[1044,['Fare']] = 13.30<count_missing_values>
TEXT_COL = 'comment_text'
VECS_PATH = Path('.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec')
 TRAIN_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'
TEST_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv'
print("Count of cabins with nan data: ")
 df_all.loc[(df_all.loc[:,'Cabin'].isnull())== True]['Name'].count()<groupby>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
df_all.groupby('Pclass' ).agg('count')[['Name','Cabin']]<groupby>
misspell_dict = {"aren't": "are not", "can't": "cannot", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not",
"hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'll": "he will", "he's": "he is",
"i'd": "I had", "i'll": "I will", "i'm": "I am", "isn't": "is not",
"it's": "it is", "it'll": "it will", "i've": "I have", "let's": "let us",
"mightn't": "might not", "mustn't": "must not", "shan't": "shall not",
"she'd": "she would", "she'll": "she will", "she's": "she is",
"shouldn't": "should not", "that's": "that is", "there's": "there is",
"they'd": "they would", "they'll": "they will", "they're": "they are",
"they've": "they have", "we'd": "we would", "we're": "we are",
"weren't": "were not", "we've": "we have", "what'll": "what will",
"what're": "what are", "what's": "what is", "what've": "what have",
"where's": "where is", "who'd": "who would", "who'll": "who will",
"who're": "who are", "who's": "who is", "who've": "who have",
"won't": "will not", "wouldn't": "would not", "you'd": "you would",
"you'll": "you will", "you're": "you are", "you've": "you have",
"'re": " are", "wasn't": "was not", "we'll": " will", "tryin'": "trying"}
( df_all.groupby('Pclass' ).agg('count')['Cabin'] / df_all.groupby('Pclass' ).agg('count')['Name'])*100<sort_values>
def _get_misspell(misspell_dict):
misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))
return misspell_dict, misspell_re
df_all[df_all['Cabin'].str.contains(' ', regex=False)== True].sort_values(by='Cabin' )<groupby>
def replace_typical_misspell(text):
misspellings, misspellings_re = _get_misspell(misspell_dict)
 def replace(match):
return misspellings[match.group(0)]
return misspellings_re.sub(replace, text )
df_cabin_expand = df_all.loc[:,'Cabin'].str.split(' ', expand=True)
 df_cabin_expand[df_cabin_expand.loc[:,1].isnull() == False].groupby([0] ).count()<filter>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']',
'>', '%', '=', '
'®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',
'½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',
'↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',
'╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
df_all[df_all['Embarked'].isnull() ]<feature_engineering>
def clean_text(x):
x = str(x)
 for punct in puncts + list(string.punctuation):
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
df_all['Embarked'] = df_all['Embarked'].fillna(method='bfill' )<groupby>
def clean_numbers(x):
return re.sub('\d+', ' ', x )
df_name_salutation = df_all.loc[:,'Name'].str.split(' ', expand=True ).copy()
df_name_salutation.groupby(1 ).count()<filter>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
df_newsal_1 = df_name_salutation[df_name_salutation[1].str.contains('.', regex=False)][1]
df_newsal_2 = df_name_salutation[df_name_salutation[2].str.contains('.', regex=False)][2]
df_newsal_3 = df_name_salutation[(df_name_salutation[3].isnull() == False)&(df_name_salutation[3].str.contains('.', regex=False)) ][3]
df_newsal_2 = df_newsal_2.rename(1)
 df_newsal_3 = df_newsal_3.rename(1)
 df_newsal = df_newsal_1.append([df_newsal_2, df_newsal_3])
 df_newsal = df_newsal.rename('Salutation')
 df_all = pd.concat([df_all,df_newsal],axis=1 )<groupby>
def load_fasttext(word_index):
embeddings_index = dict(get_coefs(*o.strip().split(' ')) for o in open(VECS_PATH))
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
df_all.groupby('Salutation' ).count()<data_type_conversions>
maxlen = 220
max_features = 100000
embed_size = 300
batch_size = 1024
train_epochs = 5
n_splits = 5
df_all.loc[:,'Family'] =(( df_all['SibSp'] > 0)|(df_all['Parch'] > 0)).replace(True, 1, inplace=False)
 df_all.loc[:,'Family'] = df_all.loc[:,'Family'].astype(int )<filter>
seed = 1024
df_familynames = df_all.loc[:,'Name'].str.split(' ', expand=True ).copy()
l_singleLastname = df_familynames[(df_familynames[0].str.contains(',', regex=False)==True)].index
l_doubleLastname = df_familynames[(df_familynames[0].str.contains(',', regex=False)==False)
 &(df_familynames[1].str.contains(',', regex=False)==True)].index
l_doubleLastnameSpec = df_familynames[(df_familynames[0].str.contains(',', regex=False)==False)
 &(df_familynames[1].str.contains(',', regex=False)==False)
 &(df_familynames[2].str.contains(',', regex=False)==True)].index
df_singleLastname = df_familynames.loc[l_singleLastname,0]
df_doubleLastname =(df_familynames.loc[l_doubleLastname,0]
+ ' ' + df_familynames.loc[l_doubleLastname,1])
 df_doubleLastnameSpec =(df_familynames.loc[l_doubleLastnameSpec,0]
+ ' ' + df_familynames.loc[l_doubleLastnameSpec,1]
+ ' ' + df_familynames.loc[l_doubleLastnameSpec,2])
 df_singleLastname = df_singleLastname.rename('Lastname')
 df_doubleLastname = df_doubleLastname.rename('Lastname')
 df_doubleLastnameSpec = df_doubleLastnameSpec.rename('Lastname')
 df_familynames['Lastname'] = df_singleLastname.append([df_doubleLastname, df_doubleLastnameSpec])
<merge>
def load_and_prec() :
train = pd.read_csv(TRAIN_DATA, index_col='id')
 test = pd.read_csv(TEST_DATA, index_col='id')
 train['comment_text'] = train['comment_text'].str.lower()
test['comment_text'] = test['comment_text'].str.lower()
train['comment_text'] = train['comment_text'].apply(replace_typical_misspell)
 test['comment_text'] = test['comment_text'].apply(replace_typical_misspell)
 train['comment_text'] = train['comment_text'].apply(clean_text)
 test['comment_text'] = test['comment_text'].apply(clean_text)
 train['comment_text'] = train['comment_text'].apply(clean_numbers)
 test['comment_text'] = test['comment_text'].apply(clean_numbers)
 train_x = train['comment_text'].fillna('_
test_x = test['comment_text'].fillna('_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x)
 train_x = pad_sequences(train_x, maxlen=maxlen)
 test_x = pad_sequences(test_x, maxlen=maxlen)
 train_y =(train['target'].values > 0.5 ).astype(int)
 np.random.seed(seed)
 train_idx = np.random.permutation(len(train_x))
train_x = train_x[train_idx]
train_y = train_y[train_idx]
return train_x, train_y, test_x, tokenizer.word_index
df_lastname_count = df_familynames.groupby('Lastname', as_index=False ).count()
df_lastname_count = df_lastname_count.drop([1,2,3,4,5,6,7,8,9,10,11,12,13], axis=1)
 df_familynames = df_familynames.drop([0,1,2,3,4,5,6,7,8,9,10,11,12,13], axis=1)
 df_familynames = df_familynames.join(df_lastname_count.set_index('Lastname'), on='Lastname')
 df_familynames = df_familynames.rename(columns = {0: "Number_of_Familymembers"} )<merge>
class Net(nn.Module):
def __init__(self, embedding_matrix):
super(Net, self ).__init__()
lstm_hidden_size = 120
gru_hidden_size = 60
self.gru_hidden_size = gru_hidden_size
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, lstm_hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(lstm_hidden_size*2, gru_hidden_size, bidirectional=True, batch_first=True)
 self.linear = nn.Linear(gru_hidden_size*6, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.unsqueeze(h_embedding.transpose(1, 2), 2)
 h_embedding = torch.squeeze(self.embedding_dropout(h_embedding)).transpose(1, 2)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, self.gru_hidden_size*2)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( hh_gru, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
df_all = pd.merge(df_all , df_familynames, right_index=True, left_index=True)
 df_all['Lastname'] = df_all['Lastname'].str.rstrip(',' )<feature_engineering>
warnings.filterwarnings('ignore' )
df_all.loc[df_all['Family'] == 0,'Number_of_Familymembers'] = 1<groupby>
with timer('load data'):
train_x, train_y, test_x, word_index = load_and_prec()
embedding_matrix = load_fasttext(word_index )
df_all.groupby(['Parch'] ).agg('count' )<groupby>
import random
import os
df_sal_distr = df_all.groupby('Salutation' ).count()
df_sal_distr.reset_index(level=0, inplace=True)
 df_sal_distr = df_sal_distr[['Salutation','Pclass']]
df_sal_distr = df_sal_distr.rename(columns = {'Pclass':"Salutation_Count"})
 df_sal_distr<concatenate>
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()
df_survivalinfo = pd.concat([df_all.loc[train_index,:], Survived], axis=1 )<groupby>
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs = 0
self.run.n_iter = 0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs+=1./self.iters
self.run.n_iter+=1
def begin_epoch(self):
self.run.n_epochs = self.epoch
self.model.train()
self.run.in_train = True
def begin_validate(self):
self.model.eval()
self.run.in_train = False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass
gp_survived_gender = df_survivalinfo.groupby(['Survived','Sex'])['Name'].count() [1]
gp_gender_survived = df_survivalinfo.groupby(['Sex','Survived'] ).count() ['Name']
gp_survived_yn = df_survivalinfo.groupby(['Survived'] ).agg('count')['Name']
gp_survival_total = df_survivalinfo.groupby(['Sex','Survived'] ).count().xs('Name', axis=1)
 sizes_suvinf = [gp_survived_yn[y] for y in range(len(gp_survived_yn)) ]
explode_suvinf =(0, 0.1)
 labels_sexinf = [gp_survived_gender.index[y] for y in range(len(gp_survived_gender.index)) ]
sizes_sexinf = [gp_survived_gender[y] for y in range(len(gp_survived_gender)) ]
explode_sexinf =(0, 0.1)
 labels_maleinf = [gp_gender_survived['male'].index[y] for y in range(len(gp_gender_survived['male'].index)) ]
sizes_maleinf = [gp_gender_survived['male'][y] for y in range(len(gp_gender_survived['male'])) ]
explode_maleinf =(0, 0.1)
 labels_femaleinf = [gp_gender_survived['female'].index[y] for y in range(len(gp_gender_survived['female'].index)) ]
sizes_femaleinf = [gp_gender_survived['female'][y] for y in range(len(gp_gender_survived['female'])) ]
explode_femaleinf =(0, 0.1)
 labels_totalsuvinf = [gp_survival_total.index[y] for y in range(len(gp_gender_survived.index)) ]
sizes_totalsuvinf = [gp_survival_total[y] for y in range(len(gp_survival_total)) ]
explode_totalsuvinf =(0.3, 0.0, 0.1, 0.0)
 font = {'color': 'black',
'weight': 'normal',
'fontsize':15
}
cmap = plt.get_cmap('tab20c')
 survivedcolor = cmap(np.array([5,1]))
survivorallcolor = cmap(np.array([9,10,0,1]))
gridsize =(4,2)
 fig1 = plt.figure(figsize=(14,10))
ax1 = plt.subplot2grid(gridsize,(0,0))
ax2 = plt.subplot2grid(gridsize,(0,1))
ax3 = plt.subplot2grid(gridsize,(1,0))
ax4 = plt.subplot2grid(gridsize,(1,1))
ax5 = plt.subplot2grid(gridsize,(2,0), colspan= 2, rowspan= 2)
 fig1.suptitle('Distributions of 891 Passengers(Trainingset)', fontsize=25)
 ax1.pie(sizes_suvinf,
explode=explode_suvinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55 
)
 ax1.axis('equal')
 ax1.legend(loc='upper left',fontsize='12',labels=('not survived', 'survived'))
ax1.set_title('Survivor Distribution', fontdict=font, fontsize=20)
 ax2.pie(sizes_sexinf,
explode=explode_sexinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=outer_colors,
labeldistance=1.15,
pctdistance=0.55)
 ax2.axis('equal')
 ax2.legend(loc='upper right', fontsize='12', labels=labels_sexinf)
 ax2.set_title('Gender Survivors', fontdict=font, fontsize=20)
 ax3.pie(sizes_maleinf,
explode=explode_maleinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax3.legend(loc='lower left', fontsize='12', labels=['not survived','survived'])
 ax3.set_title('Male Survivors', fontdict=font, fontsize=20)
 ax3.axis('equal')
 ax4.pie(sizes_femaleinf,
explode=explode_femaleinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivedcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax4.legend(loc='lower right', fontsize='12', labels=['not survived','survived'])
 ax4.set_title('Female Survivors', fontdict=font, fontsize=20)
 ax4.axis('equal')
 ax5.pie(sizes_totalsuvinf,
explode=explode_totalsuvinf,
autopct='%1.1f%%',
shadow=True, startangle=90,
colors=survivorallcolor,
labeldistance=1.15,
pctdistance=0.55)
 ax5.axis('equal')
 ax5.legend(loc='lower right', fontsize='12', labels=['female not survived','female survived','male not survived','male survived'])
 ax5.set_title('Overall Survivors', fontdict=font, fontsize=20)
 ax5.axis('equal')
 plt.show()<create_dataframe>
from collections import Iterable
ptbl = pd.DataFrame.pivot_table(df_survivalinfo, values=['Fare', 'Survived'], index=['Pclass'],
aggfunc={'Survived': ['sum'], 'Fare': [min,max,np.mean]})
 ptbl 
<create_dataframe>
def listify(o):
if o is None: return []
if isinstance(o, list): return o
if isinstance(o, str): return [o]
if isinstance(o, Iterable): return list(o)
 return [o]
df_survival_sal = df_survivalinfo.groupby(['Salutation','Survived'], as_index=False)['Name'].count()
df_survival_sal = df_survival_sal.rename(columns={"Name": "Total"})
 df_survival_sal_total = df_survival_sal.groupby('Salutation', as_index=False)['Total'].agg(sum)
 df_survival_sal_nsuv = pd.DataFrame(df_survival_sal[df_survival_sal['Survived'] == 0])
 df_survival_sal_nsuv = df_survival_sal_nsuv.rename(columns={'Total':'Total_notSurvived'})
 df_survival_sal_nsuv = df_survival_sal_nsuv[['Salutation','Total_notSurvived']]
df_survival_sal_suv = pd.DataFrame(df_survival_sal[df_survival_sal['Survived'] == 1])
 df_survival_sal_suv = df_survival_sal_suv.rename(columns={'Total':'Total_Survived'})
 df_survival_sal_suv = df_survival_sal_suv[['Salutation','Total_Survived']]
df_survival_sal_total = df_survival_sal_total.join(df_survival_sal_suv.set_index('Salutation'), on='Salutation', how='outer')
 df_survival_sal_total = df_survival_sal_total.join(df_survival_sal_nsuv.set_index('Salutation'), on='Salutation', how='outer')
 df_survival_sal_total = df_survival_sal_total.fillna(value=0)
 df_survival_sal_total<create_dataframe>
class Runner() :
def __init__(self, cbs=None, cb_funcs=None):
cbs = listify(cbs)
 for cbf in listify(cb_funcs):
cb = cbf()
setattr(self, cb.name, cb)
 cbs.append(cb)
 self.stop, self.cbs = False, [TrainEvalCallback() ] + cbs
@property
def opt(self): return self.learn.opt
@property
def model(self): return self.learn.model
@property
def loss_func(self): return self.learn.loss_func
@property
def data(self): return self.learn.data
def one_batch(self, xb, yb):
try:
self.xb, self.yb = xb, yb
self('begin_batch')
 self.pred = self.model(self.xb)
 self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb)
 self('after_loss')
 if not self.in_train: return
self.loss.backward()
self('after_backward')
 self.opt.step()
self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self, dl):
self.iters = len(dl)
 try:
for xb, yb in progress_bar(dl, leave=False): self.one_batch(xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def fit(self, epochs, learn):
self.epochs, self.learn, self.loss = epochs, learn, torch.tensor(0.)
try:
for cb in self.cbs: cb.set_runner(self)
 self('begin_fit')
 for epoch in range(epochs):
self.epoch = epoch
if not self('begin_epoch'): self.all_batches(self.data.train_dl)
 with torch.no_grad() :
if not self('begin_validate'): self.all_batches(self.data.valid_dl)
 self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.learn = None
def __call__(self, cb_name):
res = False
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res
ptbl_survived = pd.DataFrame.pivot_table(df_survivalinfo, values=['Fare', 'Age', 'Survived'], index=['Sex', 'Pclass'],
aggfunc={'Fare': np.mean,'Age': [min, max, np.mean], 'Survived': ['sum']})
 ptbl_survived<split>
class Learner() :
def __init__(self, model, opt, loss_func, data):
self.model, self.opt, self.loss_func, self.data = model, opt, loss_func, data
df_all_knn_hot = df_all.copy()
df_all_knn_hot = df_all_knn_hot.drop(['Name','Cabin','Fare','Ticket','Lastname'], axis=1)
 df_all_knn_hot = pd.get_dummies(df_all_knn_hot, columns=['Sex','Salutation','Embarked'])
 X_train, X_test, y_train, y_test = train_test_split(df_all_knn_hot.loc[train_index,:],
Survived, test_size = 0.30, random_state = 45 )<find_best_params>
def get_model(data, lr=0.005):
model = Net(embedding_matrix ).to(device)
 return model, torch.optim.Adam(model.parameters() , lr )
myList = list(range(1,50))
neighbors = list(myList)
 cv_scores = []
for k in neighbors:
knn = KNeighborsClassifier(n_neighbors=k)
 scores = cross_val_score(knn, X_train, y_train, cv=10, scoring='accuracy')
 cv_scores.append(scores.mean() )<train_model>
class DataBunch() :
def __init__(self, train_dl, valid_dl):
self.train_dl, self.valid_dl = train_dl, valid_dl
@property
def train_ds(self): return self.train_dl.dataset
@property
def valid_ds(self): return self.valid_dl.dataset
X_train = X_train
y_train = y_train
KNNC = KNeighborsClassifier(n_neighbors=3)
 KNNC.fit(X_train, y_train)
 y_pred = KNNC.predict(X_test)
 print(classification_report(y_test, y_pred, target_names=['0','1']))
print("Models accuracy score: ", accuracy_score(y_test, y_pred))<train_model>
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=seed ).split(train_x, train_y))
classes = ["will not suvive", "will survive"]
visualizer = ClassificationReport(KNNC, classes=classes, support=True)
 visualizer.fit(X_train, y_train)
 visualizer.score(X_test, y_test)
 g = visualizer.poof()<prepare_output>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
titanic_submission = pd.DataFrame({'PassengerId':df_all_knn_hot.loc[test_index,:].index,
'Survived':KNNC.predict(df_all_knn_hot.loc[test_index,:])})
 titanic_submission.PassengerId = titanic_submission.PassengerId.astype(int)
 titanic_submission.Survived = titanic_submission.Survived.astype(int)
 titanic_submission.groupby('Survived' ).count()<save_to_csv>
x_test_cuda = torch.tensor(test_x, dtype=torch.long ).to(device)
 test = torch.utils.data.TensorDataset(x_test_cuda)
 test_dl = DataLoader(test, batch_size=batch_size, shuffle=False )
titanic_submission.to_csv("titanic_submission_knn_4.csv", index=False )<split>
def get_data(train_idx, valid_idx):
x_train_ds = torch.tensor(train_x[train_idx], dtype=torch.long ).to(device)
 y_train_ds = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).to(device)
 x_val_ds = torch.tensor(train_x[valid_idx], dtype=torch.long ).to(device)
 y_val_ds = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).to(device)
 train_ds = torch.utils.data.TensorDataset(x_train_ds, y_train_ds)
 valid_ds = torch.utils.data.TensorDataset(x_val_ds, y_val_ds)
 train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
 valid_dl = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
 data = DataBunch(train_dl, valid_dl)
 return data
df_all_rf_hot = df_all.copy()
df_all_rf_hot = df_all_rf_hot.drop(['Name','Cabin','Fare','Ticket','Lastname'], axis=1)
 df_all_rf_hot = pd.get_dummies(df_all_rf_hot, columns=['Sex','Salutation','Embarked'])
 X_train, X_test, y_train, y_test = train_test_split(df_all_rf_hot.loc[train_index,:],
Survived, test_size = 0.30, random_state = 45 )<find_best_params>
loss_fn = nn.BCEWithLogitsLoss(reduction='sum' )
myList = list(range(1,30))
levels = list(myList)
 cv_scores = []
for l in levels:
rfc = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rfc, X_train, y_train, cv=5, scoring='recall')
 cv_scores.append(scores.mean() )<train_model>
def get_learner(train_idx, valid_idx):
data = get_data(train_idx, valid_idx)
 learn = Learner(*get_model(data), loss_fn, data=data)
 return learn
X_train = X_train
y_train = y_train
RFCC = RandomForestClassifier(max_depth=14,
n_estimators=5000)
 RFCC.fit(X_train, y_train)
 y_pred = RFCC.predict(X_test)
 print(classification_report(y_test, y_pred, target_names=['0','1']))
print("Models accuracy score: ", accuracy_score(y_test, y_pred))<train_model>
class AvgStats() :
def __init__(self, metrics, in_train): self.metrics, self.in_train = listify(metrics), in_train
def reset(self):
self.tot_loss, self.count = 0., 0
self.tot_mets = [0.]*len(self.metrics)
 @property
def all_stats(self): return [self.tot_loss.item() ] + self.tot_mets
@property
def avg_stats(self): return [o/self.count for o in self.all_stats]
def __repr__(self):
if not self.count: return ''
return f"{'train' if self.in_train else 'valid'}: {self.avg_stats}"
def accumulate(self, run):
bn = run.xb.shape[0]
self.tot_loss+=run.loss*bn
self.count+=bn
for i, m in enumerate(self.metrics):
self.tot_mets[i]+=m(run.pred, run.yb)*bn
classes = ["will not suvive", "will survive"]
visualizer = ClassificationReport(RFCC, classes=classes, support=True)
 visualizer.fit(X_train, y_train)
 visualizer.score(X_test, y_test)
 g = visualizer.poof()<create_dataframe>
class AvgStatsCallBack(Callback):
def __init__(self, metrics):
self.train_stats, self.valid_stats = AvgStats(metrics, True), AvgStats(metrics, False)
 def begin_epoch(self):
self.train_stats.reset()
self.valid_stats.reset()
def after_loss(self):
stats = self.train_stats if self.in_train else self.valid_stats
with torch.no_grad() : stats.accumulate(self.run)
 def after_epoch(self):
print(self.train_stats)
 print(self.valid_stats )
titanic_submission_rfc = pd.DataFrame({'PassengerId':df_all_rf_hot.loc[test_index,:].index,
'Survived':RFCC.predict(df_all_rf_hot.loc[test_index,:])})
 titanic_submission_rfc.PassengerId = titanic_submission_rfc.PassengerId.astype(int)
 titanic_submission_rfc.Survived = titanic_submission_rfc.Survived.astype(int)
 titanic_submission_rfc.groupby('Survived' ).count()<save_to_csv>
%matplotlib inline
titanic_submission_rfc.to_csv("titanic_submission_rfc_5.csv", index=False )<import_modules>
from sklearn.metrics import roc_auc_score
import scipy.special 
<define_variables>
def roc(out, y):
score = roc_auc_score(y.cpu().detach().numpy() , out.cpu().detach().numpy())
 return score
conf_performance_list = []<init_hyperparams>
class ParamScheduler(Callback):
_order = 1
def __init__(self, pname, sched_funcs): self.pname, self.sched_funcs = pname, sched_funcs
def begin_fit(self):
if not isinstance(self.sched_funcs,(list, tuple)) :
self.sched_funcs = [self.sched_funcs]*len(self.opt.param_groups)
 def set_param(self):
assert len(self.opt.param_groups)==len(self.sched_funcs)
 for pg, f in zip(self.opt.param_groups, self.sched_funcs):
pg[self.pname] = f(self.n_epochs/self.epochs)
 def begin_batch(self):
if self.in_train: self.set_param()
input_nodes = 29
hidden_nodes = 3
output_nodes = 2
learningrate = 0.4
nn_epochs = 1000<choose_model_class>
class LR_Find(Callback):
_order = 1
def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
self.max_iter, self.min_lr, self.max_lr = max_iter, min_lr, max_lr
self.best_loss = 1e9
def begin_batch(self):
if not self.in_train: return
pos = self.n_iter/self.max_iter
lr = self.min_lr*(self.max_lr/self.min_lr)**pos
for pg in self.opt.param_groups: pg['lr'] = lr
def after_step(self):
if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
raise CancelTrainException()
if self.loss<self.best_loss: self.best_loss = self.loss
titanic_nn = neuralNetwork(input_nodes, hidden_nodes, output_nodes, learningrate )<split>
run = Runner(cb_funcs=[LR_Find, Recorder] )
X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(df_all_knn_hot.loc[train_index,:], Survived, test_size = 0.15, random_state = 45 )<concatenate>
torch.cuda.empty_cache()
Xy_train_nn = pd.concat([X_train_nn, y_train_nn], axis=1 )<train_model>
gc.collect()
epochs = nn_epochs
for e in range(epochs):
for row in Xy_train_nn.itertuples() :
inputs =(np.asfarray(row[1:30])/ 29 * 0.99)+ 0.01
targets = np.zeros(output_nodes)+ 0.01
targets[int(row[30])] = 0.99
titanic_nn.train(inputs, targets)
 pass
pass 
<concatenate>
def annealer(f):
def _inner(start, end): return partial(f, start, end)
 return _inner
Xy_test_nn = pd.concat([X_test_nn, y_test_nn], axis=1 )<find_best_params>
import math
scorecard = []
matrixlist = []
for index,row in Xy_test_nn.iterrows() :
inputs = row[0:29].values
correct_label = row[29]
results = titanic_nn.query(inputs)
 label = np.argmax(results)
 print('PassengerID:', index, ' - Networks answer: ', label, ' --> Correct answer: ', correct_label)
 matrixlist.append([results, label, correct_label])
 if(label == correct_label):
scorecard.append(1)
 else:
scorecard.append(0)
 pass 
<compute_test_metric>
def pg_dicts(pgs): return [{'params': o} for o in pgs]
scorecard_array = np.array(scorecard)
 nn_accuracy = scorecard_array.sum() / scorecard_array.size
conf_performance_list.append([nn_accuracy,learningrate,hidden_nodes,epochs])
 print('Accuracy score by "75/15"-network: ', nn_accuracy )<train_model>
def cos_1cycle_anneal(start, high, end):
return [sched_cos(start, high), sched_cos(high, end)]
epochs = nn_epochs
for e in range(epochs):
for row in Xy_test_nn.itertuples() :
inputs =(np.asfarray(row[1:30])/ 29 * 0.99)+ 0.01
targets = np.zeros(output_nodes)+ 0.01
targets[int(row[30])] = 0.99
titanic_nn.train(inputs, targets)
 pass
pass<create_dataframe>
from functools import partial
titanic_submission_nn = pd.DataFrame(columns=['PassengerId','Survived'])
 for index,row in df_all_rf_hot.loc[test_index].iterrows() :
inputs = row[0:29].values
results = titanic_nn.query(inputs)
 label = np.argmax(results)
 titanic_submission_nn = titanic_submission_nn.append({'PassengerId' : index , 'Survived': label} , ignore_index=True)
 pass<data_type_conversions>
phases = [0.2, 0.8]
titanic_submission_nn.PassengerId = titanic_submission_nn.PassengerId.astype(int)
 titanic_submission_nn.Survived = titanic_submission_nn.Survived.astype(int)
 titanic_submission_nn.groupby('Survived' ).count()<save_to_csv>
scheds = combine_scheds(phases, [sched_cos(1e-4, 5e-3), sched_cos(5e-3, 1e-3)] )
titanic_submission_nn.to_csv("titanic_submission_nn_6.csv", index=False )<import_modules>
from sklearn.metrics import roc_auc_score
import tensorflow as tf<split>
def accuracy(out, yb): return(torch.argmax(out, dim=1)==yb.long() ).float().mean()
X_train_tfnn, X_test_tfnn, y_train_tfnn, y_test_tfnn = train_test_split(df_all_knn_hot.loc[train_index,:], Survived, test_size = 0.15, random_state = 45)
 X_train_tfnn = tf.keras.utils.normalize(np.asfarray(X_train_tfnn),axis= -1)
 X_test_tfnn = tf.keras.utils.normalize(np.asfarray(X_test_tfnn),axis= -1)
 y_train_tfnn = np.asfarray(y_train_tfnn)
 y_test_tfnn = np.asfarray(y_test_tfnn )<train_on_grid>
run = Runner(cb_funcs=cbfs )
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(2, activation=tf.nn.softmax))
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.fit(X_train_tfnn, y_train_tfnn, epochs = 200 )<compute_test_metric>
def train_and_eval() :
test_preds = np.zeros(len(test_x))
for fold,(train_idx, valid_idx)in enumerate(splits):
print('Fold:', fold)
 torch.cuda.empty_cache()
learn = get_learner(train_idx, valid_idx)
 gc.collect()
run = Runner(cb_funcs=cbfs)
 learn.model.train()
run.fit(4, learn)
 learn.model.eval()
test_preds_fold = np.zeros(len(test_dl.dataset))
for i,(x_batch,)in enumerate(test_dl):
with torch.no_grad() :
y_pred = learn.model(x_batch ).detach()
test_preds_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds+=test_preds_fold/len(splits)
 del(learn)
 gc.collect()
print(f'Test {fold} added')
 print('Training Completed')
 return test_preds
val_loss, val_acc = model.evaluate(X_test_tfnn, y_test_tfnn )<save_model>
def sigmoid(x): return 1/(1+np.exp(-x))
model.save('titanic_survivor_predictor.model' )<load_pretrained>
preds = train_and_eval()
new_model = tf.keras.models.load_model('titanic_survivor_predictor.model' )<import_modules>
sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 sub['prediction'] = preds
sub.reset_index(drop=False, inplace=True)
 sub.head()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
sub.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
sub.to_csv('submission.csv', index=False )