Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
num_words=50000
max_len=64
tqdm.pandas()
%matplotlib inline
print(os.listdir(".. /input"))
le_sex = LabelEncoder()
le_sex.fit(training["Sex"])
 encoded_sex_training = le_sex.transform(training["Sex"])
 training["Sex"] = encoded_sex_training
encoded_sex_testing = le_sex.transform(testing["Sex"])
 testing["Sex"] = encoded_sex_testing
le_embarked = LabelEncoder()
le_embarked.fit(training["Embarked"])
 encoded_embarked_training = le_embarked.transform(training["Embarked"])
 training["Embarked"] = encoded_embarked_training
encoded_embarked_testing = le_embarked.transform(testing["Embarked"])
 testing["Embarked"] = encoded_embarked_testing 
<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train.shape)
 print("Test shape : ",test.shape )
training["FamSize"] = training["SibSp"] + training["Parch"] + 1
testing["FamSize"] = testing["SibSp"] + testing["Parch"] + 1<feature_engineering>
def build_vocab(sentences, verbose = True):

 vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', ' huge number ', x)
 x = re.sub('[0-9]{4}', ' year ', x)
 x = re.sub('[0-9]{3}', ' number ', x)
 x = re.sub('[0-9]{2}', ' number ', x)
 return x
def clean_more(x):
x=re.sub('\s+', ' ', x ).strip()
regex = re.compile('[^a-zA-Z] ')
 return regex.sub('', x)
 def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'citicise':'criticize',
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 train["question_text"] = train["question_text"].progress_apply(lambda x: clean_text(x))
train["question_text"] = train["question_text"].progress_apply(lambda x: clean_numbers(x))
train["question_text"] = train["question_text"].progress_apply(lambda x: clean_more(x))
train["question_text"] = train["question_text"].progress_apply(lambda x: x.lower())
 train["question_text"] = train["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
sentences = train["question_text"].apply(lambda x: x.split())
 test["question_text"] = test["question_text"].progress_apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].progress_apply(lambda x: clean_numbers(x))
test["question_text"] = test["question_text"].progress_apply(lambda x: clean_more(x))
test["question_text"] = test["question_text"].progress_apply(lambda x: x.lower())
 test["question_text"] = test["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
tsentences = test["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(list(sentences)+list(tsentences))
def wordindex(vocab,n):
word_index={}
sorted_v = sorted(vocab.items() , key=operator.itemgetter(1)) [::-1]
for i in range(n-3):
word_index[sorted_v[i][0]]=i+3
return(word_index)
 word_index=wordindex(vocab,num_words)
 def zif(word):
ans=2
if(word in word_index):
ans=word_index[word]
return ans
x_train = [[zif(word)for word in sentence] for sentence in tqdm(sentences)]
x_train=pad_sequences(x_train, maxlen=max_len)
 x_test = [[zif(word)for word in sentence] for sentence in tqdm(tsentences)]
x_test=pad_sequences(x_test, maxlen=max_len)
 print({k: vocab[k] for k in list(vocab)[:5]} )
training["IsAlone"] = training.FamSize.apply(lambda x: 1 if x == 1 else 0)
 testing["IsAlone"] = testing.FamSize.apply(lambda x: 1 if x == 1 else 0 )<feature_engineering>
def wordindex(vocab,n):
word_index={}
sorted_v = sorted(vocab.items() , key=operator.itemgetter(1)) [::-1]
for i in range(n-3):
word_index[sorted_v[i][0]]=i+3
return(word_index)
 word_index=wordindex(vocab,num_words)
 def zif(word):
ans=2
if(word in word_index):
ans=word_index[word]
return ans
x_train = [[zif(word)for word in sentence] for sentence in tqdm(sentences)]
x_train=pad_sequences(x_train, maxlen=max_len)
 x_test = [[zif(word)for word in sentence] for sentence in tqdm(tsentences)]
x_test=pad_sequences(x_test, maxlen=max_len)
 y_train=train['target']
for name in training["Name"]:
training["Title"] = training["Name"].str.extract("([A-Za-z]+)\.",expand=True)
 for name in testing["Name"]:
testing["Title"] = testing["Name"].str.extract("([A-Za-z]+)\.",expand=True )<define_variables>
del train
del sentences
del tsentences
titles = set(training["Title"])
 print(titles )<define_variables>
embedding_matrix_Glove = np.zeros(( num_words, 300))
with open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt', 'r')as f:
for line in tqdm(f):
values = line.split()
word = values[0]
if(word in word_index):
try:
word_vector = np.asarray(values[1:], dtype='float32')
 except ValueError:
pass
else:
embedding_matrix_Glove[word_index[word]] = word_vector
title_list = list(training["Title"])
 frequency_titles = []
for i in titles:
frequency_titles.append(title_list.count(i))
print(frequency_titles )<create_dataframe>
embedding_matrix_Wiki = np.zeros(( num_words, 300))
with open('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec', 'r')as f:
for line in tqdm(f):
values = line.split()
word = values[0]
if(word in word_index):
try:
word_vector = np.asarray(values[1:], dtype='float32')
 except ValueError:
pass
else:
embedding_matrix_Wiki[word_index[word]] = word_vector
titles = list(titles)
 title_dataframe = pd.DataFrame({
"Titles" : titles,
"Frequency" : frequency_titles
})
 print(title_dataframe )<categorify>
def f1(y_true, y_pred):
y_pred = K.round(y_pred+0.15)
 tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return K.mean(f1 )
title_replacements = {"Mlle": "Other", "Major": "Other", "Col": "Other", "Sir": "Other", "Don": "Other", "Mme": "Other",
"Jonkheer": "Other", "Lady": "Other", "Capt": "Other", "Countess": "Other", "Ms": "Other", "Dona": "Other"}
training.replace({"Title": title_replacements}, inplace=True)
 testing.replace({"Title": title_replacements}, inplace=True)
 le_title = LabelEncoder()
le_title.fit(training["Title"])
 encoded_title_training = le_title.transform(training["Title"])
 training["Title"] = encoded_title_training
encoded_title_testing = le_title.transform(testing["Title"])
 testing["Title"] = encoded_title_testing 
<drop_column>
class Netn:
def __init__(self,em):
tweet_input = Input(shape=(max_len,), dtype='int32')
 tweet_encoder = Embedding(num_words, 300, input_length=max_len,
weights=[em], trainable=False )(tweet_input)
 X = SpatialDropout1D(0.1 )(tweet_encoder)
 X = Bidirectional(CuDNNGRU(64, return_sequences=True))(X)
 activations = CuDNNGRU(64, return_sequences=True )(X)
 attention = Dense(1, activation='tanh' )(activations)
 attention = Flatten()(attention)
 attention = Activation('softmax' )(attention)
 attention = RepeatVector(64 )(attention)
 attention = Permute([2, 1] )(attention)
 attention = Dropout(0.5 )(attention)
 X = concatenate([activations, attention])
 x=CuDNNGRU(64, return_sequences=True )(X)
 a=GlobalMaxPooling1D()(x)
 b=GlobalAveragePooling1D()(x)
 x=concatenate([a,b])
 x=Dense(64, activation='relu' )(x)
 x=Dropout(0.4 )(x)
 output = Dense(1, activation='sigmoid' )(x)
 self.model = Model(inputs=[tweet_input], outputs=[output])
 self.model.summary()
def unfreeze(self):
self.model.layers[1].trainable = True
def fit(self,b1,b2,epp,bs,**data):
self.model.compile(**data)
 x_t=x_train[b1:b2]
y_t=y_train[b1:b2]
if(epp>0):
filepath='tmp.hd5'
cp=ModelCheckpoint(filepath, monitor="val_f1",verbose=1, save_best_only=True,mode='max')
 history=self.model.fit(x_t,
y_t,
epochs=epp,
batch_size=bs,
callbacks=[cp],
validation_split=0.1)
 self.model.load_weights(filepath)
 plt.plot(history.history['f1'], label='f1 train')
 plt.plot(history.history['val_f1'], label='f1 val')
 plt.xlabel('epoche')
 plt.ylabel('f1')
 plt.legend()
plt.show()
ansz=history.history['val_f1']
mx=max(ansz)
 print('val f1 is maximal {} on a step {}'.format(mx,ansz.index(mx)+1))
def predvec(self,x_test):
return(self.model.predict(x_test))
training.drop("Name", axis = 1, inplace = True)
 testing.drop("Name", axis = 1, inplace = True )<normalization>
m1=Netn(embedding_matrix_Wiki)
scaler = StandardScaler()
ages_train = np.array(training["Age"] ).reshape(-1, 1)
 fares_train = np.array(training["Fare"] ).reshape(-1, 1)
 ages_test = np.array(testing["Age"] ).reshape(-1, 1)
 fares_test = np.array(testing["Fare"] ).reshape(-1, 1)
 training["Age"] = scaler.fit_transform(ages_train)
 training["Fare"] = scaler.fit_transform(fares_train)
 testing["Age"] = scaler.fit_transform(ages_test)
 testing["Fare"] = scaler.fit_transform(fares_test)
<import_modules>
m1.fit(0,600000,3,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=1e-3))
m1.fit(0,1250000,5,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=3e-4))
m1.unfreeze()
m1.fit(0,1250000,14,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=3e-5))
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier<import_modules>
m2=Netn(embedding_matrix_Glove )
from sklearn.metrics import make_scorer, accuracy_score<import_modules>
m2.fit(0,600000,3,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=1e-3))
m2.fit(0,1250000,5,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=3e-4))
m2.unfreeze()
m2.fit(0,1250000,14,2000,loss='binary_crossentropy', metrics=['accuracy', f1],
optimizer=Adam(lr=3e-5))
from sklearn.model_selection import GridSearchCV<import_modules>
lval=1250000
y_v=y_train.tolist() [lval:]
x_v=x_train[lval:]
from sklearn.model_selection import GridSearchCV<prepare_x_and_y>
u1=m1.predvec(x_v)
 u2=m2.predvec(x_v )
X_train = training.drop(labels=["PassengerId", "Survived"], axis=1)
 y_train = training["Survived"]
X_test = testing.drop("PassengerId", axis=1)
<split>
def qf1(t,vv):
tp=0
fp=0
fn=0
yt=y_train.tolist()
for i in range(len(vv)) :
if vv[-i]>t:
if yt[-i]==1:
tp+=1
else:
fp+=1
else:
if yt[-i]==1:
fn+=1
return(2*tp/(2*tp+fn+fp))
def best(vv):
cc=0.35
a=0
for i in range(10):
r= qf1(cc+0.01*i,vv)
 if(r>a):
ii=i
a=r
a=0
for i in range(10):
r= qf1(cc+0.01*(ii-1)+0.002*i,vv)
 if(r>a):
iii=i
a=r
print(a)
 return(cc+0.01*(ii-1)+0.002*iii)
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0 )<train_on_grid>
bt=best(( u1+u2)/2)
 print(bt )
svc_clf = SVC()
parameters_svc = {"kernel": ["rbf", "linear"], "probability": [True, False], "verbose": [True, False]}
grid_svc = GridSearchCV(svc_clf, parameters_svc, scoring=make_scorer(accuracy_score))
grid_svc.fit(X_training, y_training)
 svc_clf = grid_svc.best_estimator_
svc_clf.fit(X_training, y_training)
 pred_svc = svc_clf.predict(X_valid)
 acc_svc = accuracy_score(y_valid, pred_svc )<compute_test_metric>
def ans(v,tr):
res=np.zeros(len(v))
for i in range(len(v)) :
if v[i]>tr:
res[i]=1
return res
print("The Score for SVC is: " + str(acc_svc))<train_on_grid>
v1=m1.predvec(x_test)
 v2=m2.predvec(x_test)
 vv=ans(( v1+v2)/2,bt )
<train_on_grid><EOS> 
out = np.column_stack(( test['qid'].values,vv))
np.savetxt('submission.csv', out, header="qid,prediction",
comments="", fmt="%s,%d" )