Kaggle competition: Quora Insincere Questions Classification
df['Tname'] = df['Ticket']
df['Tset']=0
for t in df['Tname'].unique() :
if df['Surname'].loc[(df['Tname']==t)].nunique() != 1:
df['Tset'].loc[(df['Tname']==t)] = 'mixed'
else:
df['Tset'].loc[(df['Tname']==t)] = 'monotonic'
for t in df['Tname'].unique() :
if df['Surname'].loc[(df['Tname']==t)].nunique() != 1:
df['Tset'].loc[(df['Tname']==t)] = 'mixed'
else:
df['Tset'].loc[(df['Tname']==t)] = 'monotonic'<feature_engineering>
Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU,
Conv1D ,GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D,
Conv2D, MaxPool2D, concatenate,
Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, Bidirectional, 
)
 %matplotlib inline
for t in df['Ticket'].unique() :
df['Ticket'].loc[(df['Ticket']==t)] = len(df.loc[(df['Ticket']==t)])
 df['Price'] = df['Fare'] / df['Ticket']
df.rename(columns={'Ticket':'Tgroup'}, inplace=True )<drop_column>
DEBUG = False
EMBED_SIZE = 300
MAX_FEATURES = 10000 if DEBUG else 90000
SEQUENCE_LENGTH = 50
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
WIKI = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
df.drop(['Parch', 'SibSp', 'Name', 'Surname', 'Tname', 'Fare'], axis=1, inplace=True )<categorify>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 if DEBUG:
train_df = train_df[:100000]
train_df['question_text'] = train_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
DEBUG, train_df.shape, test_df.shape
label = LabelEncoder()
cols = df.dtypes[df.dtypes == 'object'].index.tolist()
for col in cols:
df[col] = label.fit_transform(df[col] )<data_type_conversions>
def load_embedding(file, tokenizer):
def split(line):
word, *arr = line.split(' ')
 return word, np.asarray(arr, dtype='float32')
 with open(file, encoding='utf8', errors='ignore')as f:
if DEBUG:
embeddings = dict(split(line)for line in tqdm(itertools.islice(f, 10000)) if len(line)> 100)
 else:
embeddings = dict(split(line)for line in tqdm(f)if len(line)> 100)
 values = np.stack(embeddings.values())
 mean = values.mean()
std = values.std()
n_words = min(MAX_FEATURES, len(tokenizer.word_index))
embedding_matrix = np.random.normal(mean, std,(n_words, EMBED_SIZE))
for word, i in tokenizer.word_index.items() :
if i < MAX_FEATURES and word in embeddings:
embedding_matrix[i] = embeddings[word]
return embedding_matrix
df['Price'] = pd.qcut(df['Price'], 4)
 df['Age'] = pd.cut(df['Age'].astype(int), 5 )<categorify>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
df['Age'] = label.fit_transform(df['Age'])
 df['Price'] = label.fit_transform(df['Price'] )<split>
def model_gru_atten_3(embedding):
input_ = Input(shape=(SEQUENCE_LENGTH,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding] )(input_)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(96, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(SEQUENCE_LENGTH )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
a = len(train_raw)
 train = df[:a]
test = df[a:]<drop_column>
class AveragedModel:
def __init__(self, tokenizer):
self.embeddings = [
load_embedding(GLOVE, tokenizer),
load_embedding(WIKI, tokenizer),
load_embedding(PARAGRAM, tokenizer),
]
self.models = [model_gru_atten_3(embedding)for embedding in self.embeddings]
def fit(self, *args, **kwargs):
for model in self.models:
model.fit(*args, **kwargs)
 def predict(self, *args, **kwargs):
outputs = [model.predict(*args, **kwargs)for model in self.models]
return np.array(outputs ).mean(axis=0 )
test.drop(['Survived'], axis=1, inplace=True)
 test_raw.shape[0] == test.shape[0]<prepare_x_and_y>
X = train.drop(['Survived'], axis=1 ).columns.to_list()
y = ['Survived']<choose_model_class>
<feature_engineering><EOS> 
best_epochs = 2
best_threshold = 0.35
X = train_df["question_text"]
y = train_df['target'].values
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(X)
 train_X = tokenizer.texts_to_sequences(X)
 train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)
 train_y = y
model = AveragedModel(tokenizer)
 model.fit(train_X, train_y, batch_size=512, epochs=best_epochs, verbose=0)
 test_X = tokenizer.texts_to_sequences(test_df["question_text"])
 test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)
 print('Tokenized test dataset')
 prediction = model.predict([test_X], batch_size=1024, verbose=0 ).flatten()
predicted_label =(prediction > best_threshold ).astype(int)
 positive = predicted_label.mean()
print(f'Tested on {model}: {positive:.3%} positive')
 out_df = pd.read_csv('.. /input/test.csv', usecols=['qid'])
 out_df = pd.DataFrame({'qid': out_df['qid'].values})
 out_df['prediction'] = predicted_label
out_df.to_csv("submission.csv", index=False )