Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<predict_on_test>
tqdm.pandas()
%matplotlib inline
py.init_notebook_mode(connected=True)
 print(os.listdir(".. /input"))
pipe_predict_xgboost = grid_pipe_xgboost.predict(X_test )<import_modules>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )
from sklearn.metrics import mean_squared_log_error<compute_test_metric>
def build_vocab(sentences,verbose=True):
vocab={}
for sentence in tqdm(sentences,disable=(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
print(np.sqrt(mean_squared_log_error(y_test,pipe_predict_xgboost)) )<import_modules>
sentences = train["question_text"].progress_apply(lambda x: x.split() ).values
vocab = build_vocab(sentences)
 print({k: vocab[k] for k in list(vocab)[:5]} )
import lightgbm as lgb<define_search_model>
news_path = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
word2vecDict= KeyedVectors.load_word2vec_format(news_path, binary=True)
pipeline_lgm = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_lgb',lgb.LGBMRegressor())
 ] )<train_model>
def check_coverage(vocab,word2vecDict):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = word2vecDict[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
%%time
pipeline_lgm.fit(X_train,y_train )<predict_on_test>
oov = check_coverage(vocab,word2vecDict )
pipeline_lgm_predict = pipeline_lgm.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,pipeline_lgm_predict)) )<define_search_space>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
params_lgb = {
'reg_lgb__learning_rate' : [.1,.2,.3,.4,.5,.6],
'reg_lgb__n_estimators' : [50,100,200],
'reg_lgb__colsample_bytree' : [.1,.2,.3,.4,.5,.6,.7],
'reg_lgb__reg_lambda' : np.arange(0,0.6,0.1)
 }<choose_model_class>
train_df["question_text"] = train["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: clean_text(x))
sentences = train["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences )
grid_pipe_lgm = RandomizedSearchCV(estimator=pipeline_lgm,param_distributions=params_lgb,cv=4,
scoring="neg_mean_squared_log_error",n_iter=15 )<train_model>
oov = check_coverage(vocab,word2vecDict )
%%time
grid_pipe_lgm.fit(X_train,y_train )<find_best_params>
for i in range(10):
print(word2vecDict.index2entity[i] )
grid_pipe_lgm.best_params_<find_best_score>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}',"
return x
train_df["question_text"] = train["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: clean_numbers(x))
grid_pipe_lgm.best_score_<predict_on_test>
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences )
lgm_pipe_predict = grid_pipe_lgm.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,lgm_pipe_predict)) )<choose_model_class>
oov = check_coverage(vocab,word2vecDict )
pipeline_elasticnet = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_en',ElasticNet())
 ] )<train_model>
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
pipeline_elasticnet.fit(X_train,y_train )<predict_on_test>
train_df["question_text"] = train["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
sentences = train["question_text"].progress_apply(lambda x: x.split())
 to_remove = ['a','to','of','and']
sentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]
vocab = build_vocab(sentences )
predicted_base_elastic_net = pipeline_elasticnet.predict(X_test)
 print(np.sqrt(mean_squared_log_error(y_test,predicted_base_elastic_net)) )<define_search_space>
oov = check_coverage(vocab,word2vecDict )
params_elasticnet = {
'reg_en__l1_ratio' : [0.2,0.4,0.6,0.8],
'reg_en__alpha' : [0.5,1,1.5,2],
}<choose_model_class>
embed_size = 300
maxlen = 200
max_features = 20000
tokenizer = Tokenizer(num_words=max_features)
 train_x,val_x=train_test_split(train_df, test_size=0.1, random_state=2018)
 train_X=train_x["question_text"].fillna("_na_" ).values
val_X=val_x["question_text"].fillna("_na_" ).values
test_X=test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_x['target'].values
val_y = val_x['target'].values
grid_pipe_elastic = RandomizedSearchCV(estimator=pipeline_elasticnet,param_distributions=params_elasticnet,cv=4,
scoring="neg_mean_squared_log_error" )<train_model>
embeddings_index = dict()
for word in word2vecDict.wv.vocab:
embeddings_index[word] = word2vecDict.word_vec(word )
%%time
grid_pipe_elastic.fit(X_train,y_train )<find_best_params>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features: continue
if word in word2vecDict:
embedding_vector = word2vecDict.get_vector(word)
 embedding_matrix[i] = embedding_vector
del word2vecDict; gc.collect()
grid_pipe_elastic.best_params_<find_best_score>
embed_size = 300
max_features = 20000
maxlen = 200
S_DROPOUT = 0.4
DROPOUT = 0.1
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size , weights=[embedding_matrix_3] )(inp)
 x = SpatialDropout1D(S_DROPOUT )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 x = Dense(16, activation="relu" )(conc)
 x = Dropout(DROPOUT )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
grid_pipe_elastic.best_score_<import_modules>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
from sklearn.linear_model import Lasso<import_modules>
embed_size = 300
max_features = 50000
maxlen = 100
train_x,val_x=train_test_split(train, test_size=0.1, random_state=42)
 train_X=train_x["question_text"].fillna("_na_" ).values
val_X=val_x["question_text"].fillna("_na_" ).values
test_X=test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_x['target'].values
val_y = val_x['target'].values
from sklearn.linear_model import Lasso<choose_model_class>
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
lasso = Lasso()<choose_model_class>
model.fit(train_X, train_y, batch_size=1024, epochs=2, validation_data=(val_X, val_y))
pipeline_lasso = Pipeline([ 
('featureunion',combined_pipeline), 
('dictifier',FunctionTransformer(func=return_dict,validate=False)) , 
('vectorizer',DictVectorizer(sort=False,sparse=False)) , 
('reg_lasso',Lasso())
 ] )<define_search_space>
pred_test_y = model.predict([val_X], batch_size=512, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_test_y>thresh ).astype(int))))
params_lasso = {
'reg_lasso__alpha' : np.logspace(-4, -3, 5),
}<choose_model_class>
pred_y = model.predict([test_X], batch_size=512, verbose=1 )
grid_pipe_lasso = RandomizedSearchCV(estimator=pipeline_lasso,param_distributions=params_lasso,cv=4,
scoring="neg_mean_squared_error" )<train_model>
pred_y =(pred_y>0.35 ).astype(int)
 out= pd.DataFrame({"qid":test["qid"].values})
 out['prediction'] = pred_y
out.to_csv("submission.csv", index=False )