Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<create_dataframe>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sns.set_style('whitegrid' )
def preprocess_data(df):
scaler = StandardScaler()
numerical_data = df[numerical_columns]
categorical_data = df[categorical_columns]
std_data_numerical = scaler.fit_transform(numerical_data)
 df_numerical = pd.DataFrame(std_data_numerical,columns=numerical_columns,index=df.index)
 std_data_categorical = pd.get_dummies(categorical_data,columns=categorical_columns)
 final_data = df_numerical.join(std_data_categorical,how='inner')
 return final_data<categorify>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
emb_arr = []
for line in tqdm(f):
try:
emb_arr.append(get_coefs(*line.strip().split(' ')))
 except Exception as e:
print(e)
 return dict(emb_arr)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = None
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )
preprocessed_train_data = preprocess_data(cleaned_train_data )<categorify>
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )
preprocessed_test_data = preprocess_data(cleaned_test_data )<split>
lengths = np.array([len(x)for x in x_train] )
train_X, val_X, train_y, val_y = train_test_split(preprocessed_train_data,labels,random_state=1 )<train_model>
np.percentile(lengths, q=95 )
estimators = [1, 2, 4, 8, 16, 32, 64, 100, 200]
for num in estimators:
model = RandomForestClassifier(n_estimators=num)
 model.fit(train_X,train_y)
 preds = model.predict(val_X)
 print("Accuracy for {} estimators is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )<find_best_params>
batch_size = 512
n_est = [5,10,20,40,100,200]
for num in n_est:
xgb_model = xgb.XGBClassifier(n_estimators=num)
 xgb_model.fit(train_X,train_y)
 preds = xgb_model.predict(val_X)
 print("Accuracy lr {} is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )<train_model>
class TextDataset(data.Dataset):
def __init__(self, text, lens, y=None):
self.text = text
self.lens = lens
self.y = y
def __len__(self):
return len(self.lens)
 def __getitem__(self, idx):
if self.y is None:
return self.text[idx], self.lens[idx]
return self.text[idx], self.lens[idx], self.y[idx]
class Collator(object):
def __init__(self, test=False, percentile=100):
self.test = test
self.percentile = percentile
def __call__(self, batch):
global MAX_LEN
if self.test:
texts, lens = zip(*batch)
 else:
texts, lens, target = zip(*batch)
 lens = np.array(lens)
 max_len = min(int(np.percentile(lens, self.percentile)) , MAX_LEN)
 texts = torch.tensor(sequence.pad_sequences(texts, maxlen=max_len), dtype=torch.long)
 if self.test:
return texts
return texts, torch.tensor(target, dtype=torch.float32 )
iters = [50,100,150,200,300,500]
for num in iters:
model = LogisticRegression(penalty='l2',max_iter=num,random_state=1,verbose=3)
 model.fit(train_X,train_y)
 preds = model.predict(val_X)
 print("Accuracy for {} iterations is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )<predict_on_test>
train_collate = Collator(percentile=100)
 train_dataset = TextDataset(x_train, lengths, y_train_torch.numpy())
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, collate_fn=train_collate )
model = LogisticRegression(penalty='l2',solver='lbfgs',random_state=1)
 model.fit(preprocessed_train_data,labels)
 preds = model.predict(preprocessed_test_data )<save_to_csv>
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method1_time =(time.time() - start_time)/ n_repeats
output = pd.DataFrame({'PassengerId': test_data.PassengerId,
'Survived': preds})
 output.to_csv('submission.csv', index=False)
 output<set_options>
class SequenceDataset(torch.utils.data.Dataset):

 def __init__(self, sequences, choose_length, other_features=None, labels=None,
indices=None, shuffle=False, batch_size=512):
super(SequenceDataset, self ).__init__()
self.sequences = np.array(sequences)
 self.lengths = np.array([len(x)for x in sequences])
 self.n_samples = len(sequences)
 self.choose_length = choose_length
self.other_features = other_features
self.labels = labels
if indices is not None:
self.indices = indices
else:
self.indices = np.arange(len(sequences))
self.batch_size = batch_size
self.shuffle = shuffle
if self.shuffle:
self._shuffle()
def __len__(self):
return math.ceil(len(self.indices)/ self.batch_size)
 def _shuffle(self):
self.indices = np.random.permutation(self.indices)
 def __getitem__(self, i):
idx = self.indices[(self.batch_size * i):(self.batch_size *(i + 1)) ]
if self.shuffle and i == len(self)- 1:
self._shuffle()
pad_length = math.ceil(self.choose_length(self.lengths[idx]))
padded_sequences = sequence.pad_sequences(self.sequences[idx], maxlen=pad_length)
 x_batch = [torch.tensor(padded_sequences, dtype=torch.long)]
if self.other_features is not None:
x_batch += [x[idx] for x in self.other_features]
if self.labels is not None:
out = x_batch, self.labels[idx]
else:
out = x_batch
return out
%matplotlib inline<import_modules>
train_dataset = SequenceDataset(x_train, lambda lengths: lengths.max() , other_features=[lengths], shuffle=False, batch_size=batch_size)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=False )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from statsmodels.graphics.gofplots import qqplot
from sklearn.preprocessing import LabelEncoder
from scipy.stats import norm, skew, boxcox_normmax<load_from_csv>
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method2_time =(time.time() - start_time)/ n_repeats
hp_train = pd.read_csv(".. /input/train.csv")
 hp_test = pd.read_csv(".. /input/test.csv" )<drop_column>
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = lengths.max()
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_train_padded.shape
train_ID = hp_train['Id']
test_ID = hp_test['Id']
hp_train.drop(['Id'], axis=1, inplace=True)
 hp_test.drop(['Id'], axis=1, inplace=True )<filter>
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
print(hp_train.shape)
 hp_train = hp_train[hp_train['GrLivArea'] < 4500]
hp_train = hp_train[hp_train['LotArea'] < 550000]
hp_train = hp_train[hp_train['TotalBsmtSF'] < 3000]
hp_train = hp_train[hp_train['1stFlrSF'] < 2500]
hp_train = hp_train[hp_train['BsmtFinSF1'] < 2000]<prepare_x_and_y>
dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 train_loader = data.DataLoader(dataset, batch_size=batch_size, collate_fn=SequenceBucketCollator(lambda x: x.max() , 0, 1, 2))
ntrain = hp_train.shape[0]
ntest = hp_test.shape[0]
y_train = hp_train.SalePrice.values
df_all = pd.concat(( hp_train, hp_test), sort=False ).reset_index(drop=True)
 df_all.drop(['SalePrice'], axis=1, inplace=True)
 print("df_all size after concatenation of train and test data is : {}".format(df_all.shape))<create_dataframe>
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method3_time =(time.time() - start_time)/ n_repeats
missing_data = pd.DataFrame({'total_missing': df_all.isnull().sum() , 'perc_missing':(df_all.isnull().sum() /len(df_all)) *100})
 len(missing_data[missing_data.total_missing>0] )<define_variables>
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
no_amen_cat = ['PoolQC','MiscFeature','Alley','Fence','GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType', 'FireplaceQu']
no_amen_num = ['GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea']<create_dataframe>
batch_size = 512
model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 model.cuda()
train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 collator = SequenceBucketCollator(lambda lengths: lengths.max() , 0, 1, 2)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=False, collate_fn=collator)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0][0].cuda())
 max_length_seq_bucketing_time = time.time() - start_time
train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 collator = SequenceBucketCollator(lambda lengths: np.percentile(lengths.numpy() , q=95), 0, 1, 2)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=False, collate_fn=collator)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0][0].cuda())
 percentile_seq_bucketing_time = time.time() - start_time
x_train_padded = torch.tensor(sequence.pad_sequences(x_train, maxlen=MAX_LEN))
train_dataset = data.TensorDataset(x_train_padded)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0].cuda())
 maxlen_static_time = time.time() - start_time
percentile_length = math.ceil(np.percentile(lengths, q=95))
x_train_padded = torch.tensor(sequence.pad_sequences(x_train, maxlen=percentile_length))
train_dataset = data.TensorDataset(x_train_padded)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0].cuda())
 percentile_static_time = time.time() - start_time
df_all1 = df_all.copy()<feature_engineering>
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
maxlen = lengths.max()
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))
for col in no_amen_cat:
df_all1[col] = df_all1[col].fillna('None' )<feature_engineering>
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )
for col in no_amen_num:
df_all1[col] = df_all1[col].fillna(0 )<categorify>
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
mode_replace = ['Electrical', 'MSZoning', 'Utilities', 'SaleType', 'KitchenQual', 'Exterior2nd', 'Exterior1st']
for col in mode_replace:
df_all1[col] = df_all1[col].fillna(df_all1[col].mode() [0] )<feature_engineering>
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )
<count_missing_values><EOS> 
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )