Kaggle competition: Whose line is it anyway?
<SOS> metric: categorizationaccuracy Kaggle data source: whose-line-is-it-anyway?<train_model>
%matplotlib inline
sns.set_style("whitegrid")
model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=3,batch_size=1024 )<predict_on_test>
train_data = pd.read_csv('.. /input/train.csv' )
pred = model.predict(X_test)
<compute_test_metric>
train_data['president'] = train_data['president'].replace('deKlerk',0)
 train_data['president'] = train_data['president'].replace('Mandela',1)
 train_data['president'] = train_data['president'].replace('Mbeki',2)
 train_data['president'] = train_data['president'].replace('Motlanthe',3)
 train_data['president'] = train_data['president'].replace('Zuma',4)
 train_data['president'] = train_data['president'].replace('Ramaphosa',5 )
thresh=0.1
prev_score=0
for i in np.arange(0.1,0.41,0.01):
score = f1_score(y_test,(pred>i))
if score>prev_score: thresh=i
prev_score=score
print('threshold at ', i, score)
 print('Best threshold: ', thresh, ' f1score: ', f1_score(y_test,(pred>thresh)) )<string_transform>
faxis = train_data.copy()
faxis.head()
test_df['question_text'] = test_df['question_text'].apply(cleanup)
 test_X = tokenizer.texts_to_sequences(test_df['question_text'].values)
 test_X = pad_sequences(test_X,maxlen=maxlen )<predict_on_test>
train_donut = []
for i, row in faxis.iterrows() :
for text in row['text'].split('.'):
train_donut.append([row['president'], text])
 train_donut = pd.DataFrame(train_donut, columns=['president', 'text'])
 train_donut
test_pred = model.predict(test_X)
 test_pred =(test_pred>thresh ).astype('int' )<save_to_csv>
train_donut['text'] = train_donut['text'].str.lower()
train_donut.head()
pd.DataFrame({'qid':test_df['qid'].values, 'prediction':np.squeeze(test_pred)} ).to_csv('submission.csv',index=False)
<import_modules>
def remove_punctuation(text):
punc = ''.join([l for l in text if l not in string.punctuation])
 return punc
train_donut['text'] = train_donut['text'].apply(remove_punctuation )
import numpy as np
import pandas as pd
import matplotlib as mp
import matplotlib.pyplot as plt
import os
from sklearn.metrics import confusion_matrix,f1_score
from sklearn.model_selection import GridSearchCV,RandomizedSearchCV,cross_validate,train_test_split,StratifiedKFold
from datetime import datetime<import_modules>
train_donut['text'] = train_donut['text'].apply(lambda x: x.replace('”',''))
train_donut['text'] = train_donut['text'].apply(lambda x: x.replace('“',''))
train_donut['text'] = train_donut['text'].apply(lambda x: x.replace('‘',''))
train_donut['text'] = train_donut['text'].apply(lambda x: x.replace('ê','e'))
train_donut['text'] = train_donut['text'].apply(lambda x: x.replace('�',''))
from keras.models import Sequential
from keras.layers import *
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model
from keras.wrappers.scikit_learn import KerasClassifier<load_from_csv>
train_donut['text'].replace(' ', np.nan, inplace=True)
 train_donut['text'].replace('', np.nan, inplace=True)
 train_donut.dropna(subset=['text'], inplace=True)
 train_donut
train=pd.read_csv(".. /input/train.csv",sep=',')
 test=pd.read_csv(".. /input/test.csv",sep=',' )<define_variables>
tokeniser = TreebankWordTokenizer()
train_donut['tokens'] = train_donut['text'].apply(tokeniser.tokenize)
 train_donut.head()
vocabulary_size=50000
vec_size=70
emb_size=300<categorify>
lemmatizer = WordNetLemmatizer()
tokenizer = Tokenizer(num_words=vocabulary_size)
 tokenizer.fit_on_texts(train.question_text)
 sequences = tokenizer.texts_to_sequences(train.question_text )<string_transform>
def donut_lemma(words, lemmatizer):
lemma = [lemmatizer.lemmatize(word)for word in words]
return lemma
tokenizer.word_index.get('trump' )<count_values>
train_donut['lemma'] = train_donut['text'].apply(donut_lemma, args=(lemmatizer,))
train_donut.head(12 )
tokenizer.word_counts.get('trump' )<concatenate>
nltk.download('stopwords')
 stop = stopwords.words('english')
 vect = CountVectorizer(stop_words=stop,
min_df=2,
max_df=0.5,
ngram_range=(1, 2))
vect.fit(train_donut['text'] )
X = pad_sequences(sequences, maxlen=vec_size )<feature_engineering>
X = vect.fit_transform(train_donut['text'])
 X = X.toarray()
Y = train_donut['president']
def get_emb_index(filepath):
embeddings_index = dict()
f = open(filepath,errors='ignore')
 for line in f:
if len(line)>100 :
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
return embeddings_index<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.001, random_state=42)
 print('X_train', X_train.shape)
 print('X_test', X_test.shape)
 print('y_train', y_train.shape)
 print('y_test', y_test.shape )
def get_emb_matrix(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
emb_mean,emb_std
embedding_matrix = np.random.normal(emb_mean, emb_std,(vocabulary_size, emb_size))
for word, index in tokenizer.word_index.items() :
if index > vocabulary_size - 1:
break
else:
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
embedding_matrix=(embedding_matrix-emb_mean)/emb_std
return embedding_matrix<prepare_x_and_y>
lrc = LogisticRegression(random_state = 0)
 lrc.fit(X_train, y_train)
 print("TRAINING SET")
 print("Accuracy: ", lrc.score(X_train, y_train))
print("Confusion Matrix:")
 print(confusion_matrix(y_train, lrc.predict(X_train)))
 print("Logistic Regression Classification Report:")
 print(classification_report(y_train, lrc.predict(X_train)))
 print("")
 print("TEST SET")
 print("Accuracy: ", lrc.score(X_test, y_test))
print("Confusion Matrix:")
 print(confusion_matrix(y_test, lrc.predict(X_test)))
 print("Logistic Regression Classification Report:")
 print(classification_report(y_test, lrc.predict(X_test)))
%%time
glove_emb_index=get_emb_index('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 glove_emb_matrix= get_emb_matrix(glove_emb_index )<prepare_x_and_y>
pres_speech = pd.read_csv('.. /input/test.csv' )
%%time
wiki_emb_index=get_emb_index('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 wiki_emb_matrix= get_emb_matrix(wiki_emb_index )<prepare_x_and_y>
pres_speech['text'] = pres_speech['text'].str.lower()
def remove_punctuation(text):
punc = ''.join([l for l in text if l not in string.punctuation])
 return punc
pres_speech['text'] = pres_speech['text'].apply(remove_punctuation)
 pres_speech['text'] = pres_speech['text'].apply(lambda x: x.replace('”',''))
pres_speech['text'] = pres_speech['text'].apply(lambda x: x.replace('“',''))
pres_speech['text'] = pres_speech['text'].apply(lambda x: x.replace('‘',''))
pres_speech['text'] = pres_speech['text'].apply(lambda x: x.replace('ê','e'))
pres_speech['text'] = pres_speech['text'].apply(lambda x: x.replace('�',''))
pres_speech['text'].replace(' ', np.nan, inplace=True)
 pres_speech['text'].replace('', np.nan, inplace=True)
 pres_speech.dropna(subset=['text'], inplace=True)
 pres_speech
%%time
paragram_emb_index=get_emb_index('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt')
 paragram_emb_matrix= get_emb_matrix(paragram_emb_index )<create_dataframe>
X_Test = vect.transform(pres_speech['text'])
 X_Test = X_Test.toarray()
Y = pres_speech['sentence']
pd.DataFrame({'glove':glove_emb_index.get('trump'),'paragram' : paragram_emb_index.get('trump'),'wikiNews':wiki_emb_index.get('trump')} ).T<groupby>
Xt_Train = vect.transform(train_donut['text'])
 Xt_Train = Xt_Train.toarray()
Y_train = train_donut['president']
avg_emb_matrix =(glove_emb_matrix+paragram_emb_matrix+wiki_emb_matrix)/3<choose_model_class>
Pred_Train = lrc.predict(Xt_Train )
%%time
def get_model_gru(dropout=0.2,gru_units=64,dense_units=16,kernel_initializer='he_normal', embedding='avg'):
inp = Input(shape=(vec_size,))
if embedding =='avg':
x = Embedding(vocabulary_size, emb_size, weights=[avg_emb_matrix] )(inp)
 elif embedding=='glove' :
x = Embedding(vocabulary_size, emb_size, weights=[glove_emb_matrix] )(inp)
 elif embedding=='paragram':
x = Embedding(vocabulary_size, emb_size, weights=[paragram_emb_matrix] )(inp)
 elif embedding =='wiki':
x = Embedding(vocabulary_size, emb_size, weights=[wiki_emb_matrix] )(inp)
 x = SpatialDropout1D(dropout )(x)
 x = Bidirectional(CuDNNGRU(gru_units, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(dense_units,kernel_initializer=kernel_initializer, activation="relu" )(x)
 x = Dropout(dropout )(x)
 x = Dense(1, kernel_initializer=kernel_initializer,activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<define_search_space>
Train_Submit = pd.DataFrame(
 {
'President': Y_train,
'Pred_president': Pred_Train,
})
 Train_Submit.head(10 )
param = {
'dropout': [0.1,0.2,0.5],
'gru_units': [16, 32, 64],
'dense_units': [16, 32, 64],
'kernel_initializer': ['he_normal','uniform'],
'embedding': ['avg','glove','paragram','wiki']
}<choose_model_class>
Pred_Pres = lrc.predict(X_Test )
model = KerasClassifier(build_fn=get_model_gru, epochs=3, batch_size=2000,verbose=False)
 rnn_model=RandomizedSearchCV(model,n_iter=10, param_distributions=param,cv=3,scoring='f1',return_train_score=True )<train_model>
Test_Submit = pd.DataFrame(
 {
'sentence': Y,
'president': Pred_Pres,
})
%%time
rnn_model.fit(X,train.target )<predict_on_test>
Test_Submit = Test_Submit.reset_index(drop=True )
pred_train=rnn_model.predict(X )<compute_test_metric>
Test_Submit.to_csv('test_data_submission_donut_squad.csv',index=False )