Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<split>
%matplotlib inline
warnings.filterwarnings('ignore')
 tqdm_notebook().pandas()
X = f_train.drop('Survived', axis = 1)
 Y = f_train.Survived
X_test = f_test
X_test = X_test.drop('PassengerId',axis = 1)
 x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size = 0.22, random_state = 0 )<choose_model_class>
import nltk
def basic_model(x_train,y_train,x_val,y_val):
model = GradientBoostingClassifier()
model.fit(x_train, y_train)
 y_pred = model.predict(x_val)
 acc_gbc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(f'Gradient Boosting Classifier : Score {acc_gbc}')
 model = RandomForestClassifier()
model.fit(x_train, y_train)
 y_pred = model.predict(x_val)
 acc_rfc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(f'Random Forest Classifier : Score {acc_rfc}')
 model = SVC()
model.fit(x_train, y_train)
 y_pred = model.predict(x_val)
 acc_svc = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(f'Support Vector Machines : Score {acc_svc}')
 model = lgbm.LGBMClassifier()
model.fit(x_train, y_train)
 y_pred = model.predict(x_val)
 acc_lgbm = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(f'LightGBM Classifier : Score {acc_lgbm}')
 model = XGBClassifier()
model.fit(x_train, y_train)
 y_pred = model.predict(x_val)
 acc_xgb = round(accuracy_score(y_pred, y_val)* 100, 2)
 print(f'XGB Classifier : Score {acc_xgb}')
 return acc_gbc, acc_rfc, acc_svc, acc_lgbm, acc_xgb<compute_train_metric>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything()
acc_gbc, acc_rfc, acc_svc, acc_lgbm, acc_xgb = basic_model(x_train, y_train, x_val, y_val )<create_dataframe>
path = '.. /input/'
df_train = pd.read_csv(path+'train.csv')
 df_test = pd.read_csv(path+'test.csv')
 print("Number of data points in training data:", df_train.shape[0])
 print("Number of data points in test data:", df_test.shape[0] )
models_basic = pd.DataFrame({
'Model': ['Gradient Boosting Classifier','Random Forest Classifier',
'Support Vector Machines', 'LightGBM Classifier',
'XGB Classifier'],
'Score Basic Model': [acc_gbc, acc_rfc, acc_svc, acc_lgbm, acc_xgb]
})
 models_basic.sort_values(by='Score Basic Model', ascending=False )<train_model>
def clean_tag(text):
if '[math]' in text:
text = re.sub('\[math\].*?math\]', '[formula]', text)
 if 'http' in text or 'www' in text:
text = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', '[url]', text)
 return text
class model_objectif(object):
def __init__(self, models, x, y):
self.models = models
self.x = x
self.y = y
def __call__(self, trial):
models, x, y = self.models, self.x, self.y
classifier_name = models
if classifier_name == "RFC":
model = RandomForestClassifier(
 n_estimators = trial.suggest_int('n_estimators', 10, 100),
criterion = trial.suggest_categorical('criterion', ['gini', 'entropy']),
max_depth = trial.suggest_int('max_depth', 1, 6),
min_samples_split = trial.suggest_int('min_samples_split', 2, 16),
max_features = trial.suggest_categorical('max_features', ['auto', 'sqrt', 'log2', None]))
model.fit(x_train, y_train)
 elif classifier_name == "SVM":
model = SVC(
 C = trial.suggest_loguniform('C', 0.1, 10),
kernel = trial.suggest_categorical('kernel', ['linear', 'rbf', 'sigmoid']),
gamma = trial.suggest_categorical('gamma', ["scale", "auto"]))
model.fit(x_train, y_train)
 elif classifier_name == "GBC":
model = GradientBoostingClassifier(
 n_estimators = trial.suggest_int('n_estimators', 10, 100),
min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10),
max_depth = trial.suggest_int('max_depth', 1, 6),
min_samples_split = trial.suggest_int('min_samples_split', 2, 16),
max_features = trial.suggest_categorical('max_features', ['auto', 'sqrt', 'log2', None]))
model.fit(x_train, y_train)
 elif classifier_name == "LGBM":
model = lgbm.LGBMClassifier(
 reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-4, 100.0),
reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-4, 100.0),
num_leaves=trial.suggest_int('num_leaves', 10, 40))
model.fit(x_train, y_train, eval_set = [(x_val, y_val)],
early_stopping_rounds=20, verbose=-1)
 elif classifier_name == "XGBC":
model = XGBClassifier(
 xgb_max_depth = trial.suggest_int('max_depth', 1, 6),
xgb_n_estimators = trial.suggest_int('n_estimators', 10, 100),
min_child_weight = trial.suggest_int('min_samples_split', 1, 20),
colsample_bytree = trial.suggest_uniform('colsample_bytree', 0.3, 1.0),
subsample = trial.suggest_uniform('colsample_bytree', 0.3, 1.0),)
 model.fit(x_train, y_train, eval_set = [(x_val, y_val)],
early_stopping_rounds=20, verbose=-1)
 y_pred = model.predict(x_val)
 acc_score = round(accuracy_score(y_pred, y_val)* 100, 2)
 return acc_score
def parameters(study_model):
print("Number of finished trials: {}".format(len(study_model.trials)))
 print("Best trial:")
 trial = study_model.best_trial
best_params = study_model.best_params
print(" Score_value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
return best_params, trial.value
def visual_study(study_model):
fig = optuna.visualization.plot_optimization_history(study_model)
 fig.show()
fig = optuna.visualization.plot_parallel_coordinate(study_model)
 fig.show()
fig = optuna.visualization.plot_slice(study_model)
 fig.show() 
<find_best_params>
contraction_mapping = {"We'd": "We had", "That'd": "That had", "AREN'T": "Are not", "HADN'T": "Had not", "Could've": "Could have", "LeT's": "Let us", "How'll": "How will", "They'll": "They will", "DOESN'T": "Does not", "HE'S": "He has", "O'Clock": "Of the clock", "Who'll": "Who will", "What'S": "What is", "Ain't": "Am not", "WEREN'T": "Were not", "Y'all": "You all", "Y'ALL": "You all", "Here's": "Here is", "It'd": "It had", "Should've": "Should have", "I'M": "I am", "ISN'T": "Is not", "Would've": "Would have", "He'll": "He will", "DON'T": "Do not", "She'd": "She had", "WOULDN'T": "Would not", "She'll": "She will", "IT's": "It is", "There'd": "There had", "It'll": "It will", "You'll": "You will", "He'd": "He had", "What'll": "What will", "Ma'am": "Madam", "CAN'T": "Can not", "THAT'S": "That is", "You've": "You have", "She's": "She is", "Weren't": "Were not", "They've": "They have", "Couldn't": "Could not", "When's": "When is", "Haven't": "Have not", "We'll": "We will", "That's": "That is", "We're": "We are", "They're": "They' are", "You'd": "You would", "How'd": "How did", "What're": "What are", "Hasn't": "Has not", "Wasn't": "Was not", "Won't": "Will not", "There's": "There is", "Didn't": "Did not", "Doesn't": "Does not", "You're": "You are", "He's": "He is", "SO's": "So is", "We've": "We have", "Who's": "Who is", "Wouldn't": "Would not", "Why's": "Why is", "WHO's": "Who is", "Let's": "Let us", "How's": "How is", "Can't": "Can not", "Where's": "Where is", "They'd": "They had", "Don't": "Do not", "Shouldn't":"Should not", "Aren't":"Are not", "ain't": "is not", "What's": "What is", "It's": "It is", "Isn't":"Is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def clean_contractions(text):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(" ")])
 return text
models = 'RFC'
objective = model_objectif(models, X, Y)
 study_RFC = optuna.create_study(direction='maximize')
 study_RFC.optimize(objective, n_trials=100 )<find_best_params>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓',
'—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━',
'⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○',
'◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐',
'├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\x97', '↺',
'∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊',
'┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´',
'↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\x93', '≧', '］',
'\x94', '∀', '♛', '\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％',
'╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭',
'┊', '╯', '☾', '➔', '∴', '\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐',
'▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\x91', '➚', '◡', '╰', '\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒',
'☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜',
'‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢',
'≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥',
'⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ',
'✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦',
'◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ',
'⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋',
'､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞',
'◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ',
'✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱',
'☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹',
'⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚',
'☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛',
'⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙',
'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘',
'˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛',
'☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑',
'✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ',
'⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘',
'⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊',
'⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷',
'◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝',
'⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢',
'≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']
def clean_punct(x):
for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
best_params_RFC, best_score_RFC = parameters(study_RFC )<find_best_params>
def data_cleaning(x):
x = clean_tag(x)
 x = clean_contractions(x)
 x = clean_punct(x)
 return x
models = 'SVM'
objective = model_objectif(models, X, Y)
 study_SVM = optuna.create_study(direction='maximize')
 study_SVM.optimize(objective, n_trials=100 )<find_best_params>
df_train['preprocessed_question_text'] = df_train['question_text'].progress_map(lambda x: data_cleaning(x))
df_test['preprocessed_question_text'] = df_test['question_text'].progress_map(lambda x: data_cleaning(x))
best_params_SVM, best_score_SVM = parameters(study_SVM )<find_best_params>
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i, word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`"
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`"
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i, letter in enumerate(word)if i == 0 or letter != word[i-1]] )
models = 'GBC'
objective = model_objectif(models, X, Y)
 study_GBC = optuna.create_study(direction='maximize')
 study_GBC.optimize(objective, n_trials=100 )<find_best_params>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english' )
best_params_GBC, best_score_GBC = parameters(study_GBC )<find_best_params>
def load_embeddings(word_dict, lemma_dict, embedding):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 if embedding == 'glove':
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
else:
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 for key in tqdm_notebook(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
models = 'XGBC'
objective = model_objectif(models, X, Y)
 study_XGBC = optuna.create_study(direction='maximize')
 study_XGBC.optimize(objective, n_trials=100 )<find_best_params>
text_list = pd.concat([df_train['preprocessed_question_text'], df_test['preprocessed_question_text']])
 num_train_data = df_train.shape[0]
best_params_XGBC, best_score_XGBC = parameters(study_XGBC )<find_best_params>
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm_notebook(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))
models = 'LGBM'
objective = model_objectif(models, X, Y)
 study_LGBM = optuna.create_study(direction='maximize')
 study_LGBM.optimize(objective, n_trials=100 )<find_best_params>
embed_size = 300
max_features = len(word_dict)+1
maxlen = 72
best_params_LGBM, best_score_LGBM = parameters(study_LGBM )<create_dataframe>
X = pad_sequences(train_word_sequences, maxlen=maxlen)
 X_test = pad_sequences(test_word_sequences, maxlen=maxlen)
 Y = df_train['target'].values
models_tuning = pd.DataFrame({
'Model': ['Gradient Boosting Classifier',
'Random Forest Classifier',
'Support Vector Machines',
'LightGBM Classifier',
'XGB Classifier'],
'Score Tuning Model': [best_score_GBC,
best_score_RFC,
best_score_SVM,
best_score_LGBM,
best_score_XGBC]
})
 model_all = pd.merge(models_basic, models_tuning, on = 'Model')
 model_all.sort_values(by='Score Tuning Model', ascending=False, inplace = True)
 model_all<choose_model_class>
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_embeddings(word_dict, lemma_dict, 'glove')
 embedding_matrix_para, nb_words = load_embeddings(word_dict, lemma_dict, 'para')
 embedding_matrix = np.mean(( 1.28*embedding_matrix_glove, 0.72*embedding_matrix_para), axis=0)
 del embedding_matrix_glove, embedding_matrix_para
gc.collect()
embedding_matrix.shape
def submit_pred(df, test_data):
model_name = df.Model.values[0]
if model_name == 'Random Forest Classifier':
model = RandomForestClassifier(**best_params_RFC)
 model.fit(x_train, y_train)
 y_pred = model.predict(test_data)
 if model_name == 'Gradient Boosting Classifier':
model = GradientBoostingClassifier(**best_params_GBC)
 model.fit(x_train, y_train)
 y_pred = model.predict(test_data)
 if model_name == 'Support Vector Machines':
model = SVC(**best_params_SVM)
 model.fit(x_train, y_train)
 y_pred = model.predict(test_data)
 if model_name == 'LightGBM Classifier':
model = lgbm.LGBMClassifier(**best_params_LGBM)
 model.fit(x_train, y_train, eval_set = [(x_val, y_val)],
early_stopping_rounds=20, verbose=-1)
 y_pred = model.predict(test_data)
 if model_name == 'XGB Classifier':
model = XGBClassifier(**best_params_XGBC)
 model.fit(x_train, y_train, eval_set = [(x_val, y_val)],
early_stopping_rounds=20, verbose=-1)
 y_pred = model.predict(test_data)
 print(f'model use {model_name}')
 return y_pred<prepare_output>
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.RandomUniform(seed=10000)
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
y_pred = submit_pred(model_all, X_test)
 submission = pd.DataFrame({
"PassengerId": f_test['PassengerId'],
"Survived": y_pred
} )<save_to_csv>
def LSTM_GRU(spatialdropout=0.2, rnn_units=64, weight_decay=0.07):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 x = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 rnn2 = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(rnn1)
 x = concatenate([rnn1, rnn2])
 x = GlobalMaxPooling1D()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=111100))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay),)
 return model
submission.to_csv('submission.csv', index=False )<save_to_csv>
def poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], input_length=maxlen, trainable=False, )(x_input)
 x = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNGRU(gru_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=111100), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 last = Lambda(lambda t: t[:, -1], name='last' )(rnn1)
 maxpool = GlobalMaxPooling1D()(rnn1)
 attn = AttentionWeightedAverage()(rnn1)
 average = GlobalAveragePooling1D()(rnn1)
 c = concatenate([last, maxpool, attn], axis=1)
 c = Reshape(( 3, -1))(c)
 c = Lambda(lambda x: K.sum(x, axis=1))(c)
 x = BatchNormalization()(c)
 x = Dense(200, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=111000))(x)
 x = Dropout(0.2, seed=1024 )(x)
 x = BatchNormalization()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=111000))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model
submission.to_csv('submission.csv', index=False )<load_from_csv>
def BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 80, 30, 12], weight_decay=0.10):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 x = SpatialDropout1D(rate=spatialdropout, seed=10000 )(emb)
 rnn = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=123000), recurrent_initializer=initializers.Orthogonal(gain=1.0, seed=123000)) )(x)
 x1 = Conv1D(filters=filters[0], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=110000))(rnn)
 x2 = Conv1D(filters=filters[1], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=120000))(rnn)
 x3 = Conv1D(filters=filters[2], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=130000))(rnn)
 x4 = Conv1D(filters=filters[3], activation='relu', kernel_size=1, padding='same', kernel_initializer=initializers.glorot_uniform(seed=140000))(rnn)
 x1 = GlobalMaxPooling1D()(x1)
 x2 = GlobalMaxPooling1D()(x2)
 x3 = GlobalMaxPooling1D()(x3)
 x4 = GlobalMaxPooling1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation='relu', kernel_initializer=initializers.glorot_uniform(seed=111000))(c)
 x = Dropout(0.2, seed=10000 )(x)
 x = BatchNormalization()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=initializers.glorot_uniform(seed=110000))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
X = pd.read_csv(".. /input/titanic/train.csv")
 X_test_full = pd.read_csv(".. /input/titanic/test.csv" )<train_model>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
print('<<Training set>>
', X.isnull().sum())
 print('---------------------------')
 print('<<Test set>>
', X_test_full.isnull().sum() )<count_missing_values>
f1, threshold = f1_smart(np.squeeze(Y[val_list]), np.squeeze(oof[val_list]))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
<define_variables><EOS> 
y_test = y_test.reshape(( -1, 1))
submission = df_test[['qid']].copy()
submission['prediction'] =(y_test>threshold ).astype(int)
 submission.to_csv('submission.csv', index=False )