Kaggle competition: Whose line is it anyway?
<SOS> metric: categorizationaccuracy Kaggle data source: whose-line-is-it-anyway?<normalization>
sns.set(style='whitegrid', palette='muted',
rc={'figure.figsize':(15,10)} )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.GRU(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(480, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED ).split(train_X, train_y))<compute_test_metric>
pres = {'deKlerk': 0,
'Mandela': 1,
'Mbeki': 2,
'Motlanthe': 3,
'Zuma': 4,
'Ramaphosa': 5}
train.replace({'president': pres}, inplace=True )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
starts = {
0: 1,
1: 1,
2: 1,
3: 12,
4: 12,
5: 5,
6: 1,
7: 1,
8: 8,
9: 9,
10: 12,
11: 14,
12: 14,
13: 15,
14: 15,
15: 15,
16: 15,
17: 15,
18: 15,
19: 15,
20: 20,
21: 1,
22: 15,
23: 20,
24: 20,
25: 15,
26: 15,
27: 20,
28: 20,
29: 15,
30: 18
}
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<categorify>
def divide_on(df, char):
sentences = []
for i, row in df.iterrows() :
for sentence in row['text'].split(char)[starts[i]:]:
sentences.append([row['president'], sentence])
 df = pd.DataFrame(sentences, columns=['president', 'text'])
 return df[df['text'] != '']
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
train = divide_on(train, '.' )
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
train['president'].value_counts()
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
train['president'].value_counts() /train.shape[0]
import os
import time
from sklearn import tree
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import GaussianNB
import numpy as np
import pandas as pd
from tqdm import tqdm
import string
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from nltk.corpus import stopwords
import seaborn as sns
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<count_missing_values>
train['sentence'] = None
test['president'] = None
df = pd.concat([train, test], axis=0, sort=False )
train.isnull().sum()<count_missing_values>
df = df[['sentence', 'text', 'president']]
train.isnull().sum()<split>
def fixup(text):
text = ''.join([char for char in text if char == '-' or char not in string.punctuation])
 text = text.replace(r'^[*-]', '')
 text = ''.join([char for char in text if not char.isdigit() ])
 text = text.lower()
text = " ".join(text.split())
 return text
df['text'] = df['text'].apply(fixup )
train, val = train_test_split(train, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train["question_text"].fillna("_na_" ).values
val_X = val["question_text"].fillna("_na_" ).values
test_X = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train['target'].values
val_y = val['target'].values<statistical_test>
df['length'] = df['text'].apply(len )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector 
<choose_model_class>
df.sort_values(by='length', ascending=False ).head(10 )
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<choose_model_class>
df.loc[3930][1]
tree = tree.DecisionTreeClassifier()
gauss = GaussianNB()<train_model>
df = df[df['length']>10]
<predict_on_test>
df.sort_values(by='length' ).head(5 )
tree.fit(train_X, train_y)
 gauss.fit(train_X,train_y)
 y_pred=tree.predict(val_X)
 y_pred_1=gauss.predict(val_X)
 print('decision tree accuracy is: ',accuracy_score(val_y, y_pred))
print('naive bayes accuracy is: ',accuracy_score(val_y, y_pred_1)) 
<save_to_csv>
df['president'].value_counts()
pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False)
 print(out_df.tail() )<feature_engineering>
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import TfidfVectorizer
train["num_words"] = train["question_text"].apply(lambda x: len(str(x ).split()))
test["num_words"] = test["question_text"].apply(lambda x: len(str(x ).split()))
train["num_unique_words"] = train["question_text"].apply(lambda x: len(set(str(x ).split())))
 test["num_unique_words"] = test["question_text"].apply(lambda x: len(set(str(x ).split())))
 train["num_chars"] = train["question_text"].apply(lambda x: len(str(x)))
 test["num_chars"] = test["question_text"].apply(lambda x: len(str(x)))
 train["num_stopwords"] = train["question_text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test["num_stopwords"] = test["question_text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train["num_punctuations"] =train['question_text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test["num_punctuations"] =test['question_text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train["mean_word_len"] = train["question_text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test["mean_word_len"] = test["question_text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
sns.violinplot(data=train,x="target", y="num_words" )<install_modules>
tfidf = TfidfVectorizer(strip_accents='unicode', ngram_range=(1,3), stop_words='english', min_df=6)
 X = tfidf.fit_transform(df['text'] ).todense()
X.shape
!pip install hyperas<load_from_csv>
tfidf.get_feature_names()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<count_missing_values>
X = pd.DataFrame(data=X, columns=tfidf.get_feature_names() )
train_df.isna().sum()<count_duplicates>
df = df.drop(columns=['text', 'length'], axis=1 )
print(train_df.duplicated(subset=['question_text'] ).sum())
 print(test_df.duplicated(subset=['question_text'] ).sum() )<feature_engineering>
X = pd.DataFrame(np.hstack(( df, X)) )
train_df['qlen'] = train_df['question_text'].apply(lambda x: len(x.split()))
test_df['qlen'] = test_df['question_text'].apply(lambda x: len(x.split()))<feature_engineering>
X.columns = ['sentence_id', 'president_id'] + tfidf.get_feature_names()
glove_embedding = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
file = open(glove_embedding)
 embedding_index={}
for line in tqdm(file):
split_values = line.split(" ")
 embedding_index[split_values[0]] = np.asarray(split_values[1:], dtype='float32')
 file.close()<feature_engineering>
train = X[pd.isnull(X['sentence_id'])]
test = X[pd.notnull(X['sentence_id'])]
def cleanup(text):
text = "".join([char for char in text if(ord(char)>=48 and ord(char)<=57)or(ord(char)>=65 and ord(char)<=90)or(ord(char)>=97 and ord(char)<=122)or(ord(char)==32)])
 text = " ".join(word if(word in embedding_index)and(word.lower() not in embedding_index)else word.lower() for word in text.split())
 return text
train = train_df.copy()
train['question_text'] = train['question_text'].apply(cleanup )<feature_engineering>
X_train = train.drop(['sentence_id', 'president_id'], axis=1)
 X_test = test.drop(['sentence_id', 'president_id'], axis=1 )
train['qlen_wostopwords'] = train['question_text'].apply(lambda x: len(x.split()))<groupby>
def one_hot_encode(label):
vec = [0, 0, 0, 0, 0, 0]
vec[label] = 1
return vec
y_train = np.vstack(row for row in train['president_id'].apply(one_hot_encode ).values )
train.groupby(by=['target'] ).mean()<define_variables>
def create_model(lyrs=[X_train.shape[1], 1028, 512, 256], act='relu', opt='Adam', dr=0.25):
model = Sequential()
model.add(Dense(lyrs[0], input_dim=X_train.shape[1], activation=act))
for i in range(1,len(lyrs)) :
model.add(Dense(lyrs[i], activation=act))
model.add(Dropout(dr))
model.add(Dense(6, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
 return model
print('Max words in train dataset: ', train.qlen_wostopwords.max())
 maxlen=100<split>
training = model.fit(X_train, y_train, epochs=5, batch_size=64, validation_split=0.2)
 val_acc = np.mean(training.history['val_acc'])
 print("
%s: %.2f%%" %('val_acc', val_acc*100))
X_train, X_test, y_train, y_test = train_test_split(train['question_text'],train['target'], test_size=0.2, random_state=42 )<train_model>
predictions = model.predict(X_test )
tokenizer = Tokenizer(num_words=50000)
 tokenizer.fit_on_texts(list(X_train))<string_transform>
pred_lbls = []
for pred in predictions:
pred = list(pred)
 max_value = max(pred)
 max_index = pred.index(max_value)
 pred_lbls.append(max_index)
 predictions = np.array(pred_lbls )
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test )<categorify>
test['president_id'] = predictions
X_train = pad_sequences(X_train,maxlen=maxlen)
 X_test = pad_sequences(X_test,maxlen=maxlen)
 y_train = y_train.values
y_test = y_test.values<define_variables>
test['president_id'].value_counts()
embedding_matrix = np.zeros(( 50000,300))
for word,i in tokenizer.word_index.items() :
if i>=50000: continue
v = embedding_index.get(word)
 if v is not None: embedding_matrix[i]=v<compute_test_metric>
submission = test[['sentence_id','president_id']]
submission.columns = ['sentence', 'president']
submission.to_csv('rnn_1.csv', index=False )
<choose_model_class><EOS> 
submission.president.value_counts()