Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<data_type_conversions>
%matplotlib inline
all_data["PoolQC"].fillna("NoPool", inplace=True)
 all_data["MiscFeature"].fillna("NOMiscFeature", inplace=True)
 all_data["Alley"].fillna("NOAlley", inplace=True)
 all_data["Fence"].fillna("NOFence", inplace=True)
 all_data["FireplaceQu"].fillna("NOFireplaceQu", inplace=True)
 all_data["GarageCond"].fillna("NOGarageCond", inplace=True)
 all_data["GarageQual"].fillna("NOGarageQual", inplace=True)
 all_data["GarageYrBlt"].fillna("NOGarageYrBlt", inplace=True)
 all_data["GarageFinish"].fillna("NOGarageFinish", inplace=True)
 all_data["GarageType"].fillna("NOGarageType", inplace=True)
 all_data["BsmtCond"].fillna("NOBsmtCond", inplace=True)
 all_data["BsmtExposure"].fillna("NOBsmtExposure", inplace=True)
 all_data["BsmtQual"].fillna("NOBsmtQual", inplace=True)
 all_data["BsmtFinType2"].fillna("NOBsmtFinType2", inplace=True)
 all_data["BsmtFinType1"].fillna("NOBsmtFinType1", inplace=True)
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
all_data["BsmtFinSF2"] = all_data.groupby("Neighborhood")["BsmtFinSF2"].transform(lambda x: x.fillna(x.median()))
all_data["BsmtFinSF1"] = all_data.groupby("Neighborhood")["BsmtFinSF1"].transform(lambda x: x.fillna(x.median()))
all_data["MasVnrType"] = all_data.groupby("Neighborhood")["MasVnrType"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["MasVnrArea"] = all_data.groupby("Neighborhood")["MasVnrArea"].transform(lambda x: x.fillna(x.mean()))
all_data["MSZoning"] = all_data.groupby("Neighborhood")["MSZoning"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["Exterior2nd"] = all_data.groupby("Neighborhood")["Exterior2nd"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["BsmtUnfSF"] = all_data.groupby("Neighborhood")["BsmtUnfSF"].transform(lambda x: x.fillna(x.median()))
all_data["TotalBsmtSF"] = all_data.groupby("Neighborhood")["TotalBsmtSF"].transform(lambda x: x.fillna(x.median()))
all_data["Exterior1st"] = all_data.groupby("Neighborhood")["Exterior1st"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["SaleType"] = all_data.groupby("Neighborhood")["SaleType"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["Electrical"] = all_data.groupby("Neighborhood")["Electrical"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["KitchenQual"] = all_data.groupby("Neighborhood")["KitchenQual"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["GarageArea"] = all_data.groupby("Neighborhood")["GarageArea"].transform(lambda x: x.fillna(x.median()))
all_data["GarageCars"] = all_data.groupby("Neighborhood")["GarageCars"].transform(lambda x: x.fillna(x.mode().values[0]))
all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data["BsmtHalfBath"].fillna(0, inplace=True)
 all_data["BsmtFullBath"].fillna(0, inplace=True)
 all_data = all_data.drop(['Utilities'], axis=1)
<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train.shape)
 print("Test shape : ",test.shape )
all_data = pd.get_dummies(all_data )<prepare_x_and_y>
EMBED_SIZE = 300
MAX_FEATURES = 100000
MAXLEN = 40
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<import_modules>
def build_vocab(texts):

 sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline
from sklearn.model_selection import KFold 
<train_on_grid>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'] )
kf = KFold(n_splits=12, random_state=42, shuffle=True)
 ridge_alphas = [1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
param_grid = {'model__alpha':ridge_alphas}
ridge = Pipeline(steps=[('standardscaler', RobustScaler()), 
('model', Ridge(max_iter=1e7)) ])
 search_ridge = GridSearchCV(ridge, param_grid,cv=kf,scoring="neg_mean_squared_error")
 search_ridge.fit(X_train, y)
 np.sqrt(search_ridge.best_score_*-1 )<prepare_output>
i = 0
for k in list(vocab):
print(k, vocab[k])
 i += 1
if i > 5:
break
cv_result = pd.DataFrame(search_ridge.cv_results_["params"])
 cv_result["score"] = np.sqrt(search_ridge.cv_results_["mean_test_score"]*-1)
 cv_result.plot(x="model__alpha",y="score");<train_on_grid>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
alphas = np.array([ 0.01,0.005,0.001,0.0001,0.00001])
 param_grid = {'model__alpha':alphas}
lasso = Pipeline(steps=[('standardscaler', RobustScaler()), 
('model', Lasso(max_iter=1e5)) ])
 search_lasso = GridSearchCV(lasso, param_grid,cv=10,scoring="neg_mean_squared_error")
 search_lasso.fit(X_train, y)
 np.sqrt(search_lasso.best_score_*-1 )<prepare_output>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove)
 print("Embedding extraction done" )
cv_result = pd.DataFrame(search_lasso.cv_results_["params"])
 cv_result["score"] = np.sqrt(search_lasso.cv_results_["mean_test_score"]*-1)
 cv_result.plot(x="model__alpha",y="score" )<create_dataframe>
def load_glove_matrix(word_index, embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
EMBED_SIZE = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, EMBED_SIZE))
for word, i in word_index.items() :
if i >= MAX_FEATURES:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
lasso_coef = pd.DataFrame(search_lasso.best_estimator_["model"].coef_,columns=["coef"])
 lasso_coef["col"] = X_train.columns<save_to_csv>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocabulary'.format(len(known_words)/len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words/(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
pred = search_lasso.best_estimator_.predict(X_test)
 df_result = pd.DataFrame(np.expm1(pred),columns=["SalePrice"])
 df_result["Id"] = test.Id
df_result.to_csv("submit.csv",index=False)
<set_options>
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
stop = set(list(stop)+ ["http","https", "s", "nt", "m"])
<load_from_csv>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )
def load_training(training_path="/kaggle/input/nlp-getting-started/train.csv"):
df = pd.read_csv(training_path)
 print(df.head(10))
return df
df = load_training()<feature_engineering>
train['question_text'] = train['question_text'].apply(lambda x: x.lower())
 test['question_text'] = test['question_text'].apply(lambda x: x.lower() )
print("-Hashtag Analysis ")
 def find_hashtags(tweet):
return ", ".join([match.group(0)[1:] for match in re.finditer(r"
def add_hashtags(df):
df['hashtag'] = df["text"].apply(lambda x: find_hashtags(x))
df['hashtag'].fillna(value="no", inplace=True)
 return df
top_N = 20
df = add_hashtags(df)
 _l = len([v for v in df.hashtag.values if isinstance(v, str)])
 print("-Number of tweets with hashtags: {}".format(_l))
print("-- Hashtag distribution in positive samples ")
 show_word_distrib(target=1, field="hashtag")
 print("-- Hashtag distribution in negative samples ")
 show_word_distrib(target=0, field="hashtag")
<feature_engineering>
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove)
 add_lower(embed_glove, vocab)
 oov_glove = check_coverage(vocab, embed_glove )
df['keyword'] = df['keyword'].map(lambda s: s.replace('%20', ' ')if isinstance(s, str)else s)
 un_KW = {kw for kw in df['keyword' ].values if isinstance(kw, str)}
tot_KW = len(df)- len(df[df["keyword" ].isna() ])
 un_LOC = {lc for lc in df['location'].values if isinstance(lc, str)}
tot_LOC = len(df)- len(df[df["location"].isna() ])
 print("Unique KW: {}".format(len(un_KW)))
 print("Out of: {}".format(tot_KW))
print("Samples with no KW: {}".format(len(df[df['keyword'].isna() ])))
 print("Unique LOC: {}".format(len(un_LOC)))
 print("Out of: {}".format(tot_LOC))
print("Samples with no Loc: {}".format(len(df[df['location'].isna() ])))
<count_values>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
len(contraction_mapping )
disaster_keywords = [kw for kw in df.loc[df.target == 1].keyword]
regular_keywords = [kw for kw in df.loc[df.target == 0].keyword]
disaster_keywords_counts = dict(pd.DataFrame(data={'x': disaster_keywords} ).x.value_counts())
 regular_keywords_counts = dict(pd.DataFrame(data={'x': regular_keywords} ).x.value_counts())
 all_keywords_counts = dict(pd.DataFrame(data={'x': df.keyword.values} ).x.value_counts())
 for keyword, _ in sorted(all_keywords_counts.items() , key=lambda x: x[1], reverse=True)[:10]:
print("> KW: {}".format(keyword))
print("--
print("--
print('--------')
<feature_engineering>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_df(df):
def remove_stopwords(text):
if text is not None:
tokens = [x for x in word_tokenize(text)if x not in stop]
return " ".join(tokens)
 else:
return None
df["text"] = df['text'].apply(lambda x : x.lower())
 def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 df['text']=df['text'].apply(lambda x : remove_URL(x))
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 df['text']=df['text'].apply(lambda x : remove_html(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df['text']=df['text'].apply(lambda x: remove_emoji(x))
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df['text']=df['text'].apply(lambda x : remove_punct(x))
df.text = df.text.replace('\s+', ' ', regex=True)
 return df
df = clean_df(df)
 print("-- Word distrig Positive Class")
 show_word_distrib(target=1, field="text")
 print("-- Word distrib Negative Class")
 show_word_distrib(target=0, field="text")
<save_to_csv>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def read_test(test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df = pd.read_csv(test_path)
 res_df = my_df[['id']]
my_df = my_df[['text']]
add_hashtags(my_df)
 my_df = clean_df(my_df)
 print("Test DF: {}".format(my_df.head(10)))
 return my_df, res_df
def dump_preds(res_df, preds, out="default"):
res_df['target'] = None
for i, p in enumerate(preds):
res_df.ix[i, 'target'] = p
res_df.to_csv(out, index = False)
 def split_data(df, _t=True):
X = df.text
if _t:
Y = df.target
le = LabelEncoder()
Y = le.fit_transform(Y)
 Y = Y.reshape(-1,1)
 return X, Y
else:
return X 
<split>
train['question_text'] = train['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
test['question_text'] = test['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
def build_tfidf_logreg(df):
my_df = df[['text','target']]
x_features = my_df.columns[0]
x_data = my_df[x_features]
Y = my_df["target"]
x_train, x_validation, y_train, y_validation = model_selection.train_test_split(
 x_data.values, Y.values, test_size=0.2, random_state=7)
 tfidf_vectorizer = TfidfVectorizer(
 analyzer='word',
tokenizer=lambda x: x,
preprocessor=lambda x: x,
token_pattern=None)
 lr = LogisticRegression()
tfidf_lr_pipe = Pipeline([('tfidf', tfidf_vectorizer),('lr', lr)])
 tfidf_lr_pipe.fit(x_train, y_train)
 return tfidf_lr_pipe
def test_tfidf_logreg(model, test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df, res_df = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 x_data = my_df["text"].values
preds = model.predict(x_data)
 return res_df

 def test_majority_model(test_path="/kaggle/input/nlp-getting-started/test.csv"):
my_df = pd.read_csv(test_path)
 res = my_df[['id']]
res['target'] = 1
res.to_csv("res_majority.csv", index = False)
 return res 
<prepare_x_and_y>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'])
 print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
X_train, y_train = split_data(df)
 test_df, res_df = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 X_test = split_data(test_df, _t=False)
 text_clf = Pipeline([('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('clf', GradientBoostingClassifier(n_estimators=100)) ,
])
<set_options>
punct_mapping = "/-'?!.,
punct_mapping += '©^®` <→°€™› ♥←×§″′Â█½à…“★”–●â►−¢²¬░¶↑±¿▾═¦║―¥▓—‹─▒：¼⊕▼▪†■’▀¨▄♫☆é¯♦¤▲è¸¾Ã⋅‘∞∙）↓、│（»，♪╩╚³・╦╣╔╗▬❤ïØ¹≤‡√'
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
%matplotlib inline
max_words = 750
max_len = 160
def process_data(X, tok=None):
if tok is None:
tok = Tokenizer(num_words=max_words)
 tok.fit_on_texts(X)
 sequences = tok.texts_to_sequences(X)
 sequences_matrix = sequence.pad_sequences(sequences,maxlen=max_len)
 return sequences_matrix, tok
def RNN() :
inputs = Input(name='inputs',shape=[max_len])
 layer = Embedding(max_words,50,input_length=max_len )(inputs)
 layer = LSTM(64 )(layer)
 layer = Dense(256,name='FC1' )(layer)
 layer = Activation('relu' )(layer)
 layer = Dropout(0.5 )(layer)
 layer = Dense(1,name='out_layer' )(layer)
 layer = Activation('sigmoid' )(layer)
 model = Model(inputs=inputs,outputs=layer)
 return model
def test_model(model, tok=None, test_path="/kaggle/input/nlp-getting-started/test.csv", cut=0.5):
my_df, res = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 X = split_data(my_df, _t=False)
 sequences_matrix, tok = process_data(X, tok=tok)
 preds = model.predict(sequences_matrix)
 normalized_preds = []
for p in preds:
if p >= cut:
normalized_preds.append(1)
 else:
normalized_preds.append(0)
 return res
def tune_cutoff(model, tok):
x_tune = df['text'].values
x_target = df[['target']]
X, Y = split_data(df)
 sequences_matrix, tok = process_data(X, tok=tok)
 preds = model.predict(sequences_matrix)
 x_target["preds"] = preds
accumulator = 0
_0 = []
max_0 = 0
_1 = []
min_1 = 0
for i, row in x_target.iterrows() :
if row['target'] == 0:
_0.append(row['preds'])
 if row['preds'] > max_0:
max_0 = row['preds']
else:
_1.append(row['preds'])
 if row['preds'] < min_1:
min_1 = row['preds']
mean_0 = sum(_0)/len(_0)
 mean_1 = sum(_1)/len(_1)
 return max(mean_0, mean_1)/min(mean_0, mean_1)
<define_variables>
puncts = {"‘": "'", "´": "'", "°": "", "€": "e", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '…': ' '}
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 return text
vocab_size = 10000
embedding_dim = 100
max_length = 50
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
validation_split = 0.10
def load_glove_vectors() :
print('Indexing word vectors.')
 GLOVE_DIR = '/kaggle/input/glove6b100dtxt/'
embeddings_index = {}
print(list(os.walk("/kaggle/input")))
 with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt')) as f:
for line in f:
word, coefs = line.split(maxsplit=1)
 coefs = np.fromstring(coefs, 'f', sep=' ')
 embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
return embeddings_index
def tokenize(df):
tokenizer = Tokenizer(num_words = vocab_size, oov_token = oov_tok)
 tokenizer.fit_on_texts(df.text)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
training_sequences = tokenizer.texts_to_sequences(df.text)
 training_padded = pad_sequences(training_sequences, maxlen = max_length,
padding = padding_type, truncating = trunc_type)
 print('Shape of the data vector is', training_padded.shape, df.target.shape)
 return training_sequences, training_padded, word_index, tokenizer
def prepare_embedding_layer(word_index, embeddings_index):
print('Preparing the embedding matrix')
 num_words = min(vocab_size, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embedding_dim))
for word, index in word_index.items() :
if index >= vocab_size:
continue
embedding_vector = embeddings_index.get(word, np.zeros(embedding_dim, dtype='float32'))
if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
embedding_layer = Embedding(num_words, embedding_dim,
embeddings_initializer = Constant(embedding_matrix),
input_length = max_length,
trainable = False)
 return embedding_layer
def model(embedding_layer):
METRICS = [
metrics.BinaryAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc')]
sequence_input = Input(shape =(max_length,))
embedded_sequences = embedding_layer(sequence_input)
 x = Bidirectional(tf.keras.layers.LSTM(64))(embedded_sequences)
 x = Dropout(0.5 )(x)
 x = Dense(32, activation = 'relu' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(1, activation = 'sigmoid' )(x)
 model = Model(sequence_input, output)
 model.compile(loss='binary_crossentropy',optimizer=tf.keras.optimizers.Adam(lr =.0002),metrics = METRICS)
 return model
def test_model(model, test_df, tokenizer):
test_sequences = tokenizer.texts_to_sequences(test_df.text)
 test_padded = pad_sequences(test_sequences, maxlen = max_length,
padding = padding_type, truncating = trunc_type)
 predictions = model.predict(test_padded)
 predictions = np.round(predictions ).astype(int ).flatten()
dump_preds(res, predictions, out="submission.csv")
<categorify>
train['question_text'] = train['question_text'].apply(lambda x: clean_special_chars(x, punct_mapping, puncts))
test['question_text'] = test['question_text'].apply(lambda x: clean_special_chars(x, punct_mapping, puncts))
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]

 kernel_size = 5
filters = 64
pool_size = 4
x = Dropout(0.1 )(sequence_output)
 x = Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1 )(x)
 x = MaxPooling1D(pool_size=pool_size )(x)
 x = Bidirectional(tf.keras.layers.LSTM(64))(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
test_df, res = read_test(test_path="/kaggle/input/nlp-getting-started/test.csv")
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(df.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test_df.text.values, tokenizer, max_len=160)
 train_labels = df.target.values
model = build_model(bert_layer, max_len=160)
 model.summary()
early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor='val_auc',
verbose=1,
patience=4,
mode='max',
restore_best_weights=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.15,
epochs=20,
batch_size=16,
callbacks=[early_stopping] 
)
 test_pred = model.predict(test_input)
 _t = []
for t in test_pred:
if t < 0.5:
_t.append(0)
 else:
_t.append(1)
 dump_preds(res, _t, out="submission.csv")
<set_options>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'])
 print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
!pip install simpletransformers
!git clone --recursive https://github.com/NVIDIA/apex.git
!pip install --upgrade --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" apex/.
 shutil.rmtree('apex')
 if os.path.exists("cache_dir"):
shutil.rmtree('cache_dir')
 if os.path.exists("outputs"):
shutil.rmtree('outputs')
 if os.path.exists("runs"):
shutil.rmtree('runs')
 seed = 98 
<import_modules>
train, val = train_test_split(train, test_size=0.2, random_state=42 )
!pip install tamnun
class DenseTransformer(TransformerMixin):
def fit(self, X, y=None, **fit_params):
return self
def transform(self, X, y=None, **fit_params):
return X.todense()
def get_models() :
models = []
tfidf_vectorizer = TfidfVectorizer(
 analyzer='word',
tokenizer=lambda x: x,
preprocessor=lambda x: x,
token_pattern=None 
)
 lr = LogisticRegression()
tfidf_lr_pipe = Pipeline([('tfidf', tfidf_vectorizer),('lr', lr)])
 dec_pipe = Pipeline([('tfidf', tfidf_vectorizer),('dt', DecisionTreeClassifier())])
 svc = Pipeline([('tfidf', tfidf_vectorizer),('svc', SVC(gamma='scale', probability=True)) ])
 gaus = Pipeline([('tfidf', tfidf_vectorizer),('to_dense', DenseTransformer()),('gaus', GaussianNB())])
 kn = Pipeline([('tfidf', tfidf_vectorizer),('kn', KNeighborsClassifier())])
 ada = Pipeline([('tfidf', tfidf_vectorizer),('ada',AdaBoostClassifier())])
 bagging = Pipeline([('tfidf', tfidf_vectorizer),('bag',BaggingClassifier(n_estimators=10)) ])
 ran_forest = Pipeline([('tfidf', tfidf_vectorizer),('ran',RandomForestClassifier(n_estimators=10)) ])
 extra_tree = Pipeline([('tfidf', tfidf_vectorizer),('extr',ExtraTreesClassifier(n_estimators=10)) ])
 gradient_boosting = Pipeline([('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('clf', GradientBoostingClassifier(n_estimators=100)) ,
])
 module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 model = build_model(bert_layer, max_len=160)
 model.summary()
models.append(model)
 models.append(dec_pipe)
 models.append(tfidf_lr_pipe)
 models.append(svc)
 models.append(gaus)
 models.append(kn)
 models.append(ada)
 models.append(bagging)
 models.append(ran_forest)
 models.append(extra_tree)
 models.append(gradient_boosting)
 return models, tokenizer
def get_out_of_fold_predictions(X, y, models, tokenizer):
	meta_X, meta_y = list() , list()
	
	kfold = StratifiedKFold(n_splits=5, shuffle=True)
 	
	for train_ix, test_ix in kfold.split(X, y):
		fold_yhats = list()
		train_X, test_X = X[train_ix], X[test_ix]
		train_y, test_y = y[train_ix], y[test_ix]
		meta_y.extend(test_y)
 		
		for i, model in enumerate(models):
			if i == 0:
				train_input = bert_encode(train_X, tokenizer, max_len=160)
 				early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_auc', verbose=1, patience=10, mode='max', restore_best_weights=True)
 				model.fit(train_input, train_y, validation_split=0.25, epochs=10,batch_size=16, callbacks=[early_stopping])
 				_test_X = bert_encode(test_X, tokenizer, max_len=160)
 				yhat = model.predict(_test_X)
 				_y = []
				for __y in yhat:
					one_prob = __y[0]
					zero_prob = 1 - one_prob
					_y.append([zero_prob, one_prob])
 				yhat = _y
			else:
				model.fit(train_X, train_y)
 				yhat = model.predict_proba(test_X)
 			
			fold_yhats.append(yhat)
 		
		meta_X.append(hstack(fold_yhats))
	return vstack(meta_X), asarray(meta_y)
 def fit_base_models(X, y, models, bert_tok):
	for i,model in enumerate(models):
		if i == 0:
			train_input = bert_encode(X, bert_tok, max_len=160)
 			early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_auc', verbose=1, patience=10, mode='max', restore_best_weights=True)
 			model.fit(train_input, y, validation_split=0.10, epochs=10,batch_size=16, callbacks=[early_stopping])
 		else:
			model.fit(X, y)
 def fit_meta_model(X, y):
	model = ExtraTreesClassifier(n_estimators=30)
 	model.fit(X, y)
 	return model
def super_learner_predictions(X, models, meta_model, bert_tok):
	meta_X = list()
	for i, model in enumerate(models):
		if i == 0:
			x = bert_encode(X, bert_tok, max_len=160)
 			yhat = model.predict(x)
 			_y = []
			for y in yhat:
				one_prob = y[0]
				zero_prob = 1 - one_prob
				_y.append([zero_prob, one_prob])
 			yhat =_y
		else:
			yhat = model.predict_proba(X)
 		meta_X.append(yhat)
 	meta_X = hstack(meta_X)
 	
	return meta_model.predict(meta_X)
<prepare_x_and_y>
xtrain = train['question_text'].fillna('_na_' ).values
xval = val['question_text'].fillna('_na_' ).values
xtest = test['question_text'].fillna('_na_' ).values
!pip install flair
def generateTrainingData(dataset, batch_size, max_length, num_classes, emb_size,
stacked_embedding):
x_batch = []
y_batch = []
while True:
data = dataset.sample(frac=1)
 for index, row in data.iterrows() :
my_sent = row["text"]
sentence = Sentence(my_sent)
 stacked_embedding.embed(sentence)
 x = []
for token in sentence:
x.append(token.embedding.cpu().detach().numpy())
 if len(x)== max_length:
break
while len(x)< max_length:
x.append(np.zeros(emb_size))
y = np.zeros(num_classes)
 y[row["target"]] = 1
x_batch.append(x)
 y_batch.append(y)
 if len(y_batch)== batch_size:
yield np.array(x_batch), np.array(y_batch)
 x_batch = []
y_batch = []
def generatePredictionData(dataset, batch_size, max_length, num_classes, emb_size, stacked_embedding):
x_batch = []
while True:
for text in dataset['text'].values:
my_sent = text
sentence = Sentence(my_sent)
 stacked_embedding.embed(sentence)
 x = []
for token in sentence:
x.append(token.embedding.cpu().detach().numpy())
 if len(x)== max_length:
break
while len(x)< max_length:
x.append(np.zeros(emb_size))
x_batch.append(x)
 if len(x_batch)== batch_size:
yield np.array(x_batch)
 x_batch = []
def get_stacked_embeddings() :
stacked_embedding = StackedEmbeddings([ELMoEmbeddings() , WordEmbeddings("en")])
 print("Stacked embedding size: {}".format(stacked_embedding.embedding_length))
embedding_size = stacked_embedding.embedding_length
return stacked_embedding, embedding_size
def declare_model(batch_size, max_len, emb_size, gru_size, num_classes):
sample = Input(batch_shape=(batch_size, max_len, emb_size))
gru_out = Bidirectional(GRU(gru_size, return_sequences=True))(sample)
 gru_out = Flatten()(gru_out)
 predictions = Dense(num_classes, activation='sigmoid' )(gru_out)
 model = Model(inputs=sample, outputs=[predictions])
 model.compile(optimizer=Adam() ,loss='binary_crossentropy', metrics=["acc"])
 print(model.summary())
 return model

<import_modules>
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(list(xtrain))
xtrain = tokenizer.texts_to_sequences(xtrain)
 xval = tokenizer.texts_to_sequences(xval)
 xtest = tokenizer.texts_to_sequences(xtest )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from scipy.stats import norm, skew<load_from_csv>
xtrain = pad_sequences(xtrain, maxlen=MAXLEN)
 xval = pad_sequences(xval, maxlen=MAXLEN)
 xtest = pad_sequences(xtest, maxlen=MAXLEN )
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
<prepare_x_and_y>
ytrain = train['target'].values
yval = val['target'].values
data = train
data = data.select_dtypes(exclude=['object'])
 data = data.dropna(axis=0)
 y = data["SalePrice"]
X = data.drop(["SalePrice"], axis=1 )<import_modules>
np.random.seed(42)
 trn_idx = np.random.permutation(len(xtrain))
val_idx = np.random.permutation(len(xval))
xtrain = xtrain[trn_idx]
ytrain = ytrain[trn_idx]
xval = xval[val_idx]
yval = yval[val_idx]
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error<count_missing_values>
print("Extracting GloVe embedding matrix")
 embedding_matrix_glove = load_glove_matrix(tokenizer.word_index, embed_glove)
 print("Embedding matrix extraction done" )
data.isnull().sum().sum()<split>
from sklearn import metrics
from sklearn.linear_model import LinearRegression
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import concatenate
X_train, X_test, y_train, y_test = train_test_split(X, y )<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
model_rf1 = RandomForestRegressor(n_estimators = 500, random_state=17)
 model_rf1.fit(X_train, y_train )<predict_on_test>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
rf1_pred = model_rf1.predict(X_test )<compute_test_metric>
def model_lstm_att(embedding_matrix):
inp = Input(shape=(MAXLEN,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 att = Attention(MAXLEN )(x)
 y = Dense(32, activation='relu' )(att)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model
mae_rf1 = mean_absolute_error(rf1_pred, y_test)
 print(mae_rf1 )<train_model>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))
pred_val_y = model.predict([xval], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([xtest], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score
model_xgb1 = XGBRegressor(n_estimators=500, learning_rate=0.1 ,random_state = 17)
 model_xgb1.fit(X_train, y_train )<predict_on_test>
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))
xgb1_pred = model_xgb1.predict(X_test )<compute_test_metric>
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0 )
mae_xgb1 = mean_absolute_error(xgb1_pred, y_test)
 print(mae_xgb1 )<save_to_csv>
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=3)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )
predictor_cols = X_train.columns
test = test[predictor_cols]
xgb1_finalpred = model_xgb1.predict(test)
 print(xgb1_finalpred)
 my_submission = pd.DataFrame({'Id': test.Id, 'SalePrice': xgb1_finalpred})
 my_submission.to_csv('xgb1_pred.csv', index=False )<load_from_csv>
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 ntrain = train.shape[0]
ntest = test.shape[0]
y = train["SalePrice"]
data = pd.concat([train, test], sort = True ).reset_index(drop=True )<drop_column>
print("Best threshold:", best_thresh, "and F1 score", thresholds[0][1] )
data.drop(['SalePrice'], axis=1, inplace=True )<create_dataframe>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )
<data_type_conversions><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )