Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<count_unique_values>
%matplotlib inline
pd.set_option('max_colwidth',400)
 print(os.listdir(".. /input"))
print(train.location.nunique() , test.location.nunique() )<categorify>
train = pd.read_csv('.. /input/jigsaw-public-files/train.csv')
 test = pd.read_csv('.. /input/jigsaw-public-files/test.csv')
 train['comment_text'] = train['comment_text'].fillna('')
 test['comment_text'] = test['comment_text'].fillna('')
 sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv' )
for col in ['keyword','location']:
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None')
 def clean_loc(x):
if x == 'None':
return 'None'
elif x == 'Earth' or x =='Worldwide' or x == 'Everywhere':
return 'World'
elif 'New York' in x or 'NYC' in x:
return 'New York'
elif 'London' in x:
return 'London'
elif 'Mumbai' in x:
return 'Mumbai'
elif 'Washington' in x and 'D' in x and 'C' in x:
return 'Washington DC'
elif 'San Francisco' in x:
return 'San Francisco'
elif 'Los Angeles' in x:
return 'Los Angeles'
elif 'Seattle' in x:
return 'Seattle'
elif 'Chicago' in x:
return 'Chicago'
elif 'Toronto' in x:
return 'Toronto'
elif 'Sacramento' in x:
return 'Sacramento'
elif 'Atlanta' in x:
return 'Atlanta'
elif 'California' in x:
return 'California'
elif 'Florida' in x:
return 'Florida'
elif 'Texas' in x:
return 'Texas'
elif 'United States' in x or 'USA' in x:
return 'USA'
elif 'United Kingdom' in x or 'UK' in x or 'Britain' in x:
return 'UK'
elif 'Canada' in x:
return 'Canada'
elif 'India' in x:
return 'India'
elif 'Kenya' in x:
return 'Kenya'
elif 'Nigeria' in x:
return 'Nigeria'
elif 'Australia' in x:
return 'Australia'
elif 'Indonesia' in x:
return 'Indonesia'
elif x in top_loc:
return x
else: return 'Others'
train['location_clean'] = train['location'].apply(lambda x: clean_loc(str(x)))
 test['location_clean'] = test['location'].apply(lambda x: clean_loc(str(x)) )<save_to_csv>
full_text = list(train['comment_text'].values)+ list(test['comment_text'].values )
leak = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv", encoding='latin_1')
 leak['target'] =(leak['choose_one']=='Relevant' ).astype(int)
 leak['id'] = leak.index
leak = leak[['id', 'target','text']]
merged_df = pd.merge(test, leak, on='id')
 sub1 = merged_df[['id', 'target']]
sub1.to_csv('submit_1.csv', index=False )<string_transform>
%%time
tk = Tokenizer(lower = True, filters='', num_words=120000)
 tk.fit_on_texts(full_text )
test_str = train.loc[417, 'text']
def clean_text(text):
text = re.sub(r'https?://\S+', '', text)
 text = re.sub(r'
',' ', text)
 text = re.sub('\s+', ' ', text ).strip()
return text
print("Original text: " + test_str)
 print("Cleaned text: " + clean_text(test_str))<feature_engineering>
train_tokenized = tk.texts_to_sequences(train['comment_text'])
 test_tokenized = tk.texts_to_sequences(test['comment_text'] )
def find_hashtags(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"
def find_mentions(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"@\w+", tweet)])or 'no'
def find_links(tweet):
return " ".join([match.group(0)[:] for match in re.finditer(r"https?://\S+", tweet)])or 'no'
def process_text(df):
df['text_clean'] = df['text'].apply(lambda x: clean_text(x))
df['hashtags'] = df['text'].apply(lambda x: find_hashtags(x))
df['mentions'] = df['text'].apply(lambda x: find_mentions(x))
df['links'] = df['text'].apply(lambda x: find_links(x))
return df
train = process_text(train)
 test = process_text(test )<feature_engineering>
max_len = 230
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )
def create_stat(df):
df['text_len'] = df['text_clean'].apply(len)
 df['word_count'] = df["text_clean"].apply(lambda x: len(str(x ).split()))
df['stop_word_count'] = df['text_clean'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df['punctuation_count'] = df['text_clean'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df['hashtag_count'] = df['hashtags'].apply(lambda x: len(str(x ).split()))
df['mention_count'] = df['mentions'].apply(lambda x: len(str(x ).split()))
df['link_count'] = df['links'].apply(lambda x: len(str(x ).split()))
df['caps_count'] = df['text_clean'].apply(lambda x: sum(1 for c in str(x)if c.isupper()))
df['caps_ratio'] = df['caps_count'] / df['text_len']
return df
train = create_stat(train)
 test = create_stat(test)
 print(train.shape, test.shape )<sort_values>
y = np.where(train['target'] >= 0.5, True, False)* 1
y_binary = to_categorical(y )
train.corr() ['target'].drop('target' ).sort_values()<define_variables>
embedding_path = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
stopwords = set(STOPWORDS)
<set_options>
embed_size = 300
max_features = 120000
wc = WordCloud().generate(stopwords)
 plt.imshow(wc, interpolation='bilinear')
 plt.axis("off")
 plt.show()<categorify>
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
features = ['keyword', 'location_clean']
encoder = ce.TargetEncoder(cols=features)
 encoder.fit(train[features],train['target'])
 train = train.join(encoder.transform(train[features] ).add_suffix('_target'))
test = test.join(encoder.transform(test[features] ).add_suffix('_target'))<prepare_x_and_y>
def build_model(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)
 inp = Input(shape =(max_len,))
x = Embedding(max_features + 1, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x1 = SpatialDropout1D(spatial_dr )(x)
 x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
 x1 = Conv1D(int(units/2), kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform' )(x_gru)
 y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
 y = Conv1D(int(units/2), kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(2, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, y_binary, batch_size = 128, epochs = 3, validation_split=0.1,
verbose = 2, callbacks = [check_point, early_stop])
 model = load_model(file_path)
 return model
vec_links = CountVectorizer(min_df = 5, analyzer = 'word', token_pattern = r'https?://\S+')
 link_vec = vec_links.fit_transform(train['links'])
 link_vec_test = vec_links.transform(test['links'])
 X_train_link = pd.DataFrame(link_vec.toarray() , columns=vec_links.get_feature_names())
 X_test_link = pd.DataFrame(link_vec_test.toarray() , columns=vec_links.get_feature_names())
 vec_men = CountVectorizer(min_df = 5)
 men_vec = vec_men.fit_transform(train['mentions'])
 men_vec_test = vec_men.transform(test['mentions'])
 X_train_men = pd.DataFrame(men_vec.toarray() , columns=vec_men.get_feature_names())
 X_test_men = pd.DataFrame(men_vec_test.toarray() , columns=vec_men.get_feature_names())
 vec_hash = CountVectorizer(min_df = 5)
 hash_vec = vec_hash.fit_transform(train['hashtags'])
 hash_vec_test = vec_hash.transform(test['hashtags'])
 X_train_hash = pd.DataFrame(hash_vec.toarray() , columns=vec_hash.get_feature_names())
 X_test_hash = pd.DataFrame(hash_vec_test.toarray() , columns=vec_hash.get_feature_names())
 print(X_train_link.shape, X_train_men.shape, X_train_hash.shape )<sort_values>
model = build_model(lr = 1e-3, lr_d = 0, units = 128, spatial_dr = 0.1)
 pred = model.predict(X_test, batch_size = 1024, verbose = 1)[:, 1]
hash_rank =(X_train_hash.transpose().dot(train['target'])/ X_train_hash.sum(axis=0)).sort_values(ascending=False)
 print('Hashtags with which 100% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==1].index))
print('Total: ' + str(len(hash_rank[hash_rank==1])))
 print('Hashtags with which 0% of Tweets are disasters: ')
 print(list(hash_rank[hash_rank==0].index))
print('Total: ' + str(len(hash_rank[hash_rank==0])) )<categorify>
def build_model1(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)
 inp = Input(shape =(max_len,))
x = Embedding(max_features + 1, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x1 = SpatialDropout1D(dr )(x)
 x = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
 y = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(2, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, y_binary, batch_size = 128, epochs = 3, validation_split=0.1,
verbose = 2, callbacks = [check_point, early_stop])
 model = load_model(file_path)
 return model
<merge><EOS> 
def build_model2(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)
 inp = Input(shape =(max_len,))
x = Embedding(max_features + 1, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x = SpatialDropout1D(dr )(x)
 x = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x)
 x = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(units * 4, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(units * 4, activation='relu' )(hidden)])
 result = Dense(2, activation='sigmoid' )(hidden)
 model = Model(inputs = inp, outputs = result)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, y_binary, batch_size = 128, epochs = 3, validation_split=0.1,
verbose = 2, callbacks = [check_point, early_stop])
 model = load_model(file_path)
 return model