Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<train_model>
tqdm.pandas()
model.fit(X, y, batch_size = 32, nb_epoch = 100 )<predict_on_test>
def set_seed(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 set_seed(2411)
 SEED = 42
num_partitions = 10
num_cores = psutil.cpu_count()
print('number of cores:', num_cores)
 def df_parallelize_run(df, func):
df_split = np.array_split(df, num_partitions)
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
y_pred = model.predict(test)
 y_final =(y_pred > 0.5 ).astype(int ).reshape(test.shape[0] )<save_to_csv>
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
glove_path = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')
sample_sub['Survived']= y_final
sample_sub.to_csv("submit.csv", index=False)
 sample_sub.head()<set_options>
spaces = ['\u200b', '\u200e', '\u202a', '\u202c', '\ufeff', '\uf0d8', '\u2061', '\x10', '\x7f', '\x9d', '\xad', '\xa0']
def remove_space(text):

 for space in spaces:
text = text.replace(space, ' ')
 text = text.strip()
text = re.sub('\s+', ' ', text)
 return text
def remove_diacritics(s):
return ''.join(c for c in normalize('NFKD', s.replace('ø', 'o' ).replace('Ø', 'O' ).replace('⁻', '-' ).replace('₋', '-'))
if category(c)!= 'Mn')
 special_punc_mappings = {"—": "-", "–": "-", "_": "-", '”': '"', "″": '"', '“': '"', '•': '.', '−': '-',
"’": "'", "‘": "'", "´": "'", "`": "'", '\u200b': ' ', '\xa0': ' ','،':'','„':'',
'…': '...', '\ufeff': ''}
def clean_special_punctuations(text):
for punc in special_punc_mappings:
if punc in text:
text = text.replace(punc, special_punc_mappings[punc])
 text = remove_diacritics(text)
 return text
def clean_number(text):
text = re.sub(r'(\d+ )([a-zA-Z])', '\g<1> \g<2>', text)
 text = re.sub(r'(\d+ )(th|st|nd|rd)', '\g<1>\g<2> ', text)
 text = re.sub(r'(\d+),(\d+)', '\g<1>\g<2>', text)
 return text
regular_punct = list(string.punctuation)
 extra_punct = [
',', '.', '"', ':', ')', '(', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤',
':)', ':)', ':-)', '(:', '(:', '(-:', ':')',
':D', ': D', ':-D', 'xD', 'x-D', 'XD', 'X-D',
'<3', ':*',
';-)', ';)', ';-D', ';D', '(;', '(-;',
':-(', ':(', ':(', ''):', ')-:',
'-- :','(', ':'(', ':"('',]
def handle_emojis(text):
text = re.sub(r'(:\s?\)|:-\)|\(\s?:|\(-:|:'\)) ', ' EMO_POS ', text)
 text = re.sub(r'(:\s?D|:-D|x-?D|X-?D)', ' EMO_POS ', text)
 text = re.sub(r'(<3|:\*)', ' EMO_POS ', text)
 text = re.sub(r'(;-?\)|;-?D|\(-?;)', ' EMO_POS ', text)
 text = re.sub(r'(:\s?\(|:-\(|\)\s?:|\)-:)', ' EMO_NEG ', text)
 text = re.sub(r'(:,\(|:'\(|:"\() ', ' EMO_NEG ', text)
 return text
def stop(text):
text = " ".join([w.lower() for w in text.split() ])
 stop_words = stopwords.words('english')
 words = [w for w in text.split() if not w in stop_words]
return " ".join(words)
 all_punct = list(set(regular_punct + extra_punct))
all_punct.remove('-')
 all_punct.remove('.')
 def clean_repeat_words(text):
text = re.sub(r"(I|i )(I|i)+ng", "ing", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+y", "lly", text)
 text = re.sub(r"(A|a )(A|a )(A|a)+", "a", text)
 text = re.sub(r"(C|c )(C|c )(C|c)+", "cc", text)
 text = re.sub(r"(D|d )(D|d )(D|d)+", "dd", text)
 text = re.sub(r"(E|e )(E|e )(E|e)+", "ee", text)
 text = re.sub(r"(F|f )(F|f )(F|f)+", "ff", text)
 text = re.sub(r"(G|g )(G|g )(G|g)+", "gg", text)
 text = re.sub(r"(I|i )(I|i )(I|i)+", "i", text)
 text = re.sub(r"(K|k )(K|k )(K|k)+", "k", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+", "ll", text)
 text = re.sub(r"(M|m )(M|m )(M|m)+", "mm", text)
 text = re.sub(r"(N|n )(N|n )(N|n)+", "nn", text)
 text = re.sub(r"(O|o )(O|o )(O|o)+", "oo", text)
 text = re.sub(r"(P|p )(P|p )(P|p)+", "pp", text)
 text = re.sub(r"(Q|q )(Q|q)+", "q", text)
 text = re.sub(r"(R|r )(R|r )(R|r)+", "rr", text)
 text = re.sub(r"(S|s )(S|s )(S|s)+", "ss", text)
 text = re.sub(r"(T|t )(T|t )(T|t)+", "tt", text)
 text = re.sub(r"(V|v )(V|v)+", "v", text)
 text = re.sub(r"(Y|y )(Y|y )(Y|y)+", "y", text)
 text = re.sub(r"plzz+", "please", text)
 text = re.sub(r"(Z|z )(Z|z )(Z|z)+", "zz", text)
 text = re.sub(r"(-+|\.+)", " ", text)
 return text
def spacing_punctuation(text):

 for punc in all_punct:
if punc in text:
text = text.replace(punc, f' {punc} ')
 return text
def preprocess(text):

 text = remove_space(text)
 text = clean_special_punctuations(text)
 text = handle_emojis(text)
 text = clean_number(text)
 text = spacing_punctuation(text)
 text = clean_repeat_words(text)
 text = remove_space(text)
 return text
mispell_dict = {'😉':'wink','😂':'joy','😀':'stuck out tongue', 'theguardian':'the guardian','deplorables':'deplorable', 'theglobeandmail':'the globe and mail', 'justiciaries': 'justiciary','creditdation': 'Accreditation','doctrne':'doctrine','fentayal': 'fentanyl','designation-': 'designation','CONartist' : 'con-artist','Mutilitated' : 'Mutilated','Obumblers': 'bumblers','negotiatiations': 'negotiations','dood-': 'dood','irakis' : 'iraki','cooerate': 'cooperate','COx':'cox','racistcomments':'racist comments','envirnmetalists': 'environmentalists',}
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def correct_contraction(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
tqdm.pandas()
def text_clean_wrapper(df):
df["comment_text"] = df["comment_text"].astype('str' ).transform(preprocess)
 df['comment_text'] = df['comment_text'].transform(lambda x: correct_spelling(x, mispell_dict))
df['comment_text'] = df['comment_text'].transform(lambda x: correct_contraction(x, contraction_mapping))
return df
train = df_parallelize_run(train, text_clean_wrapper)
 test = df_parallelize_run(test, text_clean_wrapper)
 gc.enable()
del mispell_dict, all_punct, special_punc_mappings, regular_punct, extra_punct
gc.collect()
df_test = pd.read_csv(".. /input/titanic/test.csv")
 df_train = pd.read_csv(".. /input/titanic/train.csv")
 def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test] 
<count_missing_values>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix
total_missing_train = df_train.isnull().sum().sort_values(ascending=False)
 percent_1 = df_train.isnull().sum() /df_train.isnull().count() *100
percent_2 =(round(percent_1, 1)).sort_values(ascending=False)
 train_missing_data = pd.concat([total_missing_train, percent_2], axis=1, keys=['Total', '%'])
 print(total_missing_train)
 print('_'*25)
 train_missing_data.head(5 )<sort_values>
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect()
total_missing_test = df_test.isnull().sum().sort_values(ascending=False)
 percent_3 = df_test.isnull().sum() /df_test.isnull().count() *100
percent_4 =(round(percent_3, 1)).sort_values(ascending=False)
 test_missing_data = pd.concat([total_missing_test, percent_4], axis=1, keys=['Total', '%'])
 print(total_missing_test)
 print('_'*25)
 test_missing_data.head(5 )<categorify>
embeddings_index1 = load_embeddings()
embedding_matrix1 = build_matrix(word_index, embeddings_index1)
 del embeddings_index1
gc.collect()
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {} '.format(pclass, sex, age_by_pclass_sex[sex][pclass].astype(int)))
 df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median())) 
<filter>
embeddings_index2 = load_embeddings(glove_path)
 embedding_matrix2 = build_matrix(word_index, embeddings_index2)
 del embeddings_index2
gc.collect()
df_all[df_all['Embarked'].isnull() ]<data_type_conversions>
embedding_matrix = embedding_matrix1*0.6 + embedding_matrix2*0.4
df_all['Embarked'] = df_all['Embarked'].fillna('S')
<filter>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
df_all[df_all['Fare'].isnull() ]<categorify>
gc.collect()
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'])['Fare'].median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )<count_values>
import keras.layers as L
from keras.models import Model
from keras.optimizers import Adam
data1=df_train.copy()
data1['Family_size'] = data1['SibSp'] + data1['Parch'] +1
data1['Family_size'].value_counts().sort_values(ascending=False )<groupby>
def LSTM_ATT_BN(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNGRU(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 x = L.Dense(128,activation='relu' )(x)
 x = L.BatchNormalization()(x)
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model
df_all['Deck'] = df_all['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 df_all_decks = df_all.groupby(['Deck', 'Pclass'] ).count().drop(columns=['Survived', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Embarked', 'Cabin', 'PassengerId',
'Ticket'] ).rename(columns={'Name': 'Count'})
 df_all_decks<groupby>
def get_pclass_dist(df):
deck_counts = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {}, 'G': {}, 'M': {}, 'T': {}}
decks = df.columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = df[deck][pclass][0]
deck_counts[deck][pclass] = count
except KeyError:
deck_counts[deck][pclass] = 0
df_decks = pd.DataFrame(deck_counts)
 deck_percentages = {}
for col in df_decks.columns:
deck_percentages[col] = [(count / df_decks[col].sum())* 100 for count in df_decks[col]]
return deck_counts, deck_percentages,df_decks
all_deck_count, all_deck_per,df_decks_return = get_pclass_dist(df_all_decks)
 print(df_decks_return)
 print("_"*25)
 all_deck_per<feature_engineering>
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'<count_values>
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()<split>
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
print(df_test.isnull().sum())
 print('-'*25 )<concatenate>
def model_gru_atten_3() :
inp = L.Input(shape=(maxlen,))
x = L.Embedding(len(word_index)+ 1, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = L.Bidirectional(L.CuDNNGRU(128, return_sequences=True))(x)
 x = L.Bidirectional(L.CuDNNGRU(100, return_sequences=True))(x)
 x = L.Bidirectional(L.CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = L.Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
df_all = concat_df(df_train, df_test)
 df_all.head()<feature_engineering>
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 10
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )<feature_engineering>
outputs = []
df_all['Age'] = pd.qcut(df_all['Age'], 10 )<categorify>
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
 model = model_gru_atten_3()
model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es])
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5
outputs.append([test_preds, '3 GRU w/ atten'])
 roc_auc_score(y_train>0.5,oof_preds )
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )<feature_engineering>
pred_test= np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0 )
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1<split>
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = pred_test
submission.reset_index(drop=False, inplace=True)
 submission.head()
df_train,df_test= divide_df(df_all)
 dfs=[df_train,df_test]<define_variables>
submission.to_csv('submission.csv', index=False )
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA<feature_engineering>
submission.to_csv('submission.csv', index=False )