Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<predict_on_test>
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input"))
em_preds = elastic_model.predict(X_test)
 lasso_preds = lasso_model.predict(X_test)
 stack_gen_preds = stack_gen_model.predict(X_test)
 lgbm_preds = lgbm_model.predict(X_test)
 gbr_preds = gbr_model.predict(X_test )<compute_test_metric>
nlp = spacy.load('en_core_web_sm')
 doc = nlp(u'A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels')
 for token in doc:
print(token.text, token.pos_, token.tag_, token.dep_ )
print('RMSE is:
', mean_squared_error(y_test,stack_gen_preds))<compute_test_metric>
def word_locate(sentence, location):
count_words = 0
count_chars = 2
for word in sentence.split() :
count_words += 1
if location == count_chars:
return word, count_words
count_chars += len(word)
 count_chars += 1
stack_preds_1 =(( 0.1*em_preds)+(0.1*lasso_preds)+(0.4 * gbr_preds)
 +(0.2*lgbm_preds)+(0.2*stack_gen_preds))
print('The accuracy of the stack Regressor is',r2_score(y_test,stack_preds_1))
print('RMSE is:
', mean_squared_error(y_test,stack_preds_1)) 
<concatenate>
def curr_prev_sentence(sentence, loc):
current_sentence = ""
prev_sentence = ""
detect = 0
count = 0
for char in sentence:
count += 1
current_sentence += char
if char == "." and detect == 0:
prev_sentence = current_sentence
current_sentence = ""
if char == "." and detect == 1:
return current_sentence, prev_sentence
if count == loc:
detect = 1
feats = test.select_dtypes(include=[np.number] ).interpolate().dropna()
feats = test.drop([ 'Id'], axis=1 )<predict_on_test>
def find_subject(sentence):
doc = nlp(sentence)
 for token in doc:
if token.dep_ == "nsubj":
return token.text
return "none"
em_preds = elastic_model.predict(feats)
 lasso_preds = lasso_model.predict(feats)
 stack_gen_preds = stack_gen_model.predict(feats)
 lgbm_preds = lgbm_model.predict(feats)
 gbr_preds = gbr_model.predict(feats )<prepare_output>
text1 = "After a few years of almost no work -- although he was a guest star on Lou Grant and Charlie's Angels in the late 1970s, he once summed up the 1970s as 'I cried and did a lot of gardening' -- he was hired in 1979 for his best-known role, self-made millionaire Palmer Cortlandt on ABC's long-running soap opera All My Children.Initially hired for only one year, he remained on contract through 2009.For much of his first decade on the show, Palmer was a ruthless villain, totally possessive of his daughter, Nina and violently threatening his ex-wife Daisy with being attacked by dobermans when she came back from the dead."
stack_preds=(( 0.1*em_preds)+(0.1*lasso_preds)+(0.4 * gbr_preds)
 +(0.2*lgbm_preds)+(0.2*stack_gen_preds))<predict_on_test>
text2 = "When onlookers expressed doubt, claiming that the Proctor family was well regarded in the community, the girl promptly came out of her trance and told them it was all for 'sport'.On March 29, 1692, Abigail Williams and Mercy Lewis again said they were being tormented by Elizabeth's spectre.A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels, and said she saw Elizabeth's spectre as well as John's."
final_predictions = np.exp(stack_preds)
 print("Original predictions are:
", stack_preds[:5], "
")
 print("Final predictions are:
", final_predictions[:5] )<save_to_csv>
word, loc = word_locate(text2, 360)
 print(word, " ", loc )
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['SalePrice'] = final_predictions
submission.to_csv('submission1.csv', index=False )<set_options>
with open('.. /input/gap-other-training-data/gap-training.tsv')as tsvfile:
reader = csv.DictReader(tsvfile, dialect='excel-tab')
 train_X = []
train_y = []
tr_label_A_list = []
tr_label_B_list = []
for row in reader:
text = row['Text']
text = text.lower()
new_textA = text
labelA = 0
labelB = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['A-offset']):
new_textA = new_textA[:char_idx+check] + "person_loc " + new_textA[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textA = new_textA[:char_idx+check] + "pronom_loc " + new_textA[char_idx+check:]
check = 11
if row['A-coref'] == 'TRUE':
labelA = 1
train_X.append(new_textA)
 train_y.append(labelA)
 new_textB = text
label = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['B-offset']):
new_textB = new_textB[:char_idx+check] + "person_loc " + new_textB[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textB = new_textB[:char_idx+check] + "pronom_loc " + new_textB[char_idx+check:]
check = 11
if row['B-coref'] == 'TRUE':
labelB = 1
train_X.append(new_textB)
 train_y.append(labelB)
 tr_label_A_list.append(labelA)
 tr_label_B_list.append(labelB )
warnings.filterwarnings('ignore')
 %matplotlib inline 
<load_from_csv>
with open('.. /input/gap-other-training-data/gap-training.tsv')as tsvfile:
reader = csv.DictReader(tsvfile, dialect='excel-tab')
 testA_X = []
testA_y = []
testB_X = []
testB_y = []
label_A_list = []
label_B_list = []
for row in reader:
text = row['Text']
text = text.lower()
new_textA = text
labelA = 0
labelB = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['A-offset']):
new_textA = new_textA[:char_idx+check] + "person_loc " + new_textA[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textA = new_textA[:char_idx+check] + "pronom_loc " + new_textA[char_idx+check:]
check = 11
if row['A-coref'] == 'TRUE':
labelA = 1
testA_X.append(new_textA)
 testA_y.append(labelA)
 new_textB = text
label = 0
check = 0
for char_idx in range(len(text)) :
if char_idx == int(row['B-offset']):
new_textB = new_textB[:char_idx+check] + "person_loc " + new_textB[char_idx+check:]
check = 11
if char_idx == int(row['Pronoun-offset']):
new_textB = new_textB[:char_idx+check] + "pronom_loc " + new_textB[char_idx+check:]
check = 11
if row['B-coref'] == 'TRUE':
labelB = 1
testB_X.append(new_textB)
 testB_y.append(labelB)
 label_A_list.append(labelA)
 label_B_list.append(labelB )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
maxlen = 120
embed_size = 300
max_features = 5000
df_train['SalePrice'] = np.log1p(df_train['SalePrice'] )<create_dataframe>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer_list = list(train_X)
 tokenizer.fit_on_texts(tokenizer_list)
 train_X = tokenizer.texts_to_sequences(train_X)
 testA_X = tokenizer.texts_to_sequences(testA_X)
 testB_X = tokenizer.texts_to_sequences(testB_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 testA_X = pad_sequences(testA_X, maxlen=maxlen)
 testB_X = pad_sequences(testB_X, maxlen=maxlen )
data = pd.concat([df_train['SalePrice'], df_train['1stFlrSF']], axis=1)
 f, ax = plt.subplots(figsize=(8, 6))
fig = sns.regplot(x='1stFlrSF', y="SalePrice", data=data )<drop_column>
word_index = tokenizer.word_index
max_features = len(word_index )
df_train = df_train[df_train['Id'] != 692][df_train['Id'] != 1183]
df_train = df_train.drop(df_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<12.5)].index)
 df_train = df_train.drop(df_train[(df_train['LotArea']>150000)].index)
 df_train = df_train.drop(df_train[(df_train['GarageArea']>1200)&(df_train['SalePrice']<12.5)].index )<drop_column>
!wget http://nlp.stanford.edu/data/glove.840B.300d.zip
del df_all['Utilities']<data_type_conversions>
with zipfile.ZipFile("glove.840B.300d.zip","r")as zip_ref:
zip_ref.extractall()
!ls 'cased_glove_300'
df_all["PoolQC"] = df_all["PoolQC"].fillna("None")
 df_all["MiscFeature"] = df_all["MiscFeature"].fillna("None")
 df_all["Alley"] = df_all["Alley"].fillna("None")
 df_all["Fence"] = df_all["Fence"].fillna("None")
 df_all["FireplaceQu"] = df_all["FireplaceQu"].fillna("None")
 df_all['BsmtQual'] = df_all['BsmtQual'].fillna('None')
 df_all['BsmtCond'] = df_all['BsmtCond'].fillna('None')
 df_all['BsmtExposure'] = df_all['BsmtExposure'].fillna('None')
 df_all['BsmtFinType1'] = df_all['BsmtFinType1'].fillna('None')
 df_all['BsmtFinType2'] = df_all['BsmtFinType2'].fillna('None')
 df_all['GarageType'] = df_all['GarageType'].fillna('None')
 df_all['GarageFinish'] = df_all['GarageFinish'].fillna('None')
 df_all['GarageQual'] = df_all['GarageQual'].fillna('None')
 df_all['GarageCond'] = df_all['GarageCond'].fillna('None' )<data_type_conversions>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if word == "pronom_loc":
embedding_vector = np.ones(300)*0.85
if word == "person_loc":
embedding_vector = np.ones(300)*0.25
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
df_all['BsmtFinSF1'] = df_all['BsmtFinSF1'].fillna(0)
 df_all['BsmtFinSF2'] = df_all['BsmtFinSF2'].fillna(0)
 df_all['BsmtUnfSF'] = df_all['BsmtUnfSF'].fillna(0)
 df_all['TotalBsmtSF'] = df_all['TotalBsmtSF'].fillna(0)
 df_all['BsmtFullBath'] = df_all['BsmtFullBath'].fillna(0)
 df_all['BsmtHalfBath'] = df_all['BsmtHalfBath'].fillna(0)
 df_all['MasVnrArea'] = df_all['MasVnrArea'].fillna(0)
 df_all['GarageYrBlt'] = df_all['GarageYrBlt'].fillna(0)
 df_all['GarageCars'] = df_all['GarageCars'].fillna(0)
 df_all['GarageArea'] = df_all['GarageArea'].fillna(0 )<categorify>
def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel )
df_all['MSZoning'] = df_all['MSZoning'].fillna(df_all['MSZoning'].mode() [0])
 df_all['Exterior1st'] = df_all['Exterior1st'].fillna(df_all['Exterior1st'].mode() [0])
 df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna(df_all['Exterior2nd'].mode() [0])
 df_all['MasVnrType'] = df_all['MasVnrType'].fillna(df_all['MasVnrType'].mode() [0])
 df_all['Electrical'] = df_all['Electrical'].fillna(df_all['Electrical'].mode() [0])
 df_all['KitchenQual'] = df_all['KitchenQual'].fillna(df_all['KitchenQual'].mode() [0])
 df_all['Functional'] = df_all['Functional'].fillna(df_all['Functional'].mode() [0])
 df_all['SaleType'] = df_all['SaleType'].fillna(df_all['SaleType'].mode() [0] )<feature_engineering>
def dupli_row(x, num_copies, axis):
y = x
for i in range(num_copies-1):
y = np.concatenate(( y,x), axis= axis)
 return y
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
class IsLayer(Layer):
def __init__(self, **kwargs):
super(IsLayer, self ).__init__(**kwargs)
 def build(self, input_shape):
self.W = self.add_weight(name='W',
shape=(input_shape[1], 1),
initializer='uniform',
trainable=True)
 super(IsLayer, self ).build(input_shape)
 def call(self, x):
x_W = K.dot(x, self.W)
 x_new = x*x_W
return x_new
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1] )
df_all['MSSubClass'] = df_all['MSSubClass'].apply(str)
 df_all['OverallCond'] = df_all['OverallCond'].astype(str )<categorify>
class AttentionWithContext(Layer):

 def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.u_regularizer = regularizers.get(u_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.u_constraint = constraints.get(u_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(AttentionWithContext, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 self.u = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
 super(AttentionWithContext, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
 if self.bias:
uit += self.b
uit = K.tanh(uit)
 ait = dot_product(uit, self.u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1]
one_hot_encoding = df_all.copy()
one_hot_encoding = pd.get_dummies(one_hot_encoding)
 one_hot_encoding.iloc[:,36:50].head()<drop_column>
train_y = np.asarray(train_y )
del one_hot_encoding;<categorify>
embedding_matrix = load_glove(word_index )
label_encoding = df_all.copy()
for i in categorical_features:
label_encoding[i], indexer = pd.factorize(label_encoding[i])
 label_encoding.iloc[:,20:30].head()<drop_column>
inp1 = Input(shape=(maxlen,))
model1_out = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp1)
 model1_out = Bidirectional(CuDNNGRU(256, return_sequences=True))(model1_out)
 model1_out = AttentionWithContext()(model1_out)
 model1_out = Dropout(0.1 )(model1_out)
 merged_out = Dense(32, activation="relu" )(model1_out)
 merged_out = Dropout(0.1 )(merged_out)
 merged_out = Dense(1, activation="sigmoid" )(merged_out)
 model = Model(inputs=inp1, outputs=merged_out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
del label_encoding;<categorify>
model.fit(train_X, train_y, batch_size=512, epochs=20, validation_data=(train_X, train_y))
frequency_encoding_all = df_all.copy()
def frequency_encoding(frame, col):
freq_encoding = frame.groupby([col] ).size() /frame.shape[0]
freq_encoding = freq_encoding.reset_index().rename(columns={0:'{}_Frequency'.format(col)})
 return frame.merge(freq_encoding, on=col, how='left')
 for col in categorical_features:
frequency_encoding_all = frequency_encoding(frequency_encoding_all, col)
 frequency_encoding_all = frequency_encoding_all.drop(categorical_features,axis=1, inplace=False)
 frequency_encoding_all.iloc[:,20:30].head()<drop_column>
pred_valA_y = model.predict(testA_X, batch_size=512, verbose=1)
 pred_valB_y = model.predict(testB_X, batch_size=512, verbose=1 )
del frequency_encoding_all<compute_train_metric>
print("Test score A:", log_loss(pred_valA_y,testA_y))
n_folds = 5
def rmsle_cv(model,df):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df.values)
 rmse= np.sqrt(-cross_val_score(model, df.values, y_reg, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<categorify>
print("Test score B:", log_loss(pred_valB_y,testB_y))
one_hot_encoding = df_all.copy()
del one_hot_encoding['SalePrice']
one_hot_encoding = pd.get_dummies(one_hot_encoding)
 one_hot_encoding_train = one_hot_encoding[:len_train]
one_hot_encoding_test = one_hot_encoding[len_train:]
del one_hot_encoding
model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model,one_hot_encoding_train)
 print("One-hot encoding(5-folds)LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
kfold_lgb_model = model.fit(one_hot_encoding_train, y_reg)
 train_prediction = model.predict(one_hot_encoding_train)
 prediction = np.expm1(kfold_lgb_model.predict(one_hot_encoding_test.values))
submission = pd.DataFrame({'Id': Id, 'SalePrice': prediction})
 submission.to_csv('OH_5FOLD.csv', index=False )<compute_train_metric>
out_df = pd.DataFrame({"ID":test_ids} )
n_folds = one_hot_encoding_train.shape[0]
def rmsle_cv(model,df):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df.values)
 rmse= np.sqrt(-cross_val_score(model, df.values, y_reg, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
out_df['A'] = [max(float(val),0.33)for val in list(pred_valA_y)]
out_df['B'] = [max(float(val),0.33)for val in list(pred_valB_y)]
model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model,one_hot_encoding_train)
 print("One-hot encoding(LOOCV)LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
loovc_lgb_model = model.fit(one_hot_encoding_train, y_reg)
 train_prediction = model.predict(one_hot_encoding_train)
 prediction = np.expm1(loovc_lgb_model.predict(one_hot_encoding_test.values))
submission = pd.DataFrame({'Id': Id, 'SalePrice': prediction})
 submission.to_csv('OH_LOOCV.csv', index=False )<compute_train_metric>
out_df['NEITHER'] = [max(( 1-float(val)) ,0.33)for val in list(pred_valA_y)]
<compute_train_metric><EOS> 
out_df.to_csv("submission.csv", index=False )