Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
plt.style.use('seaborn')
 sns.set(font_scale=2.5)
 warnings.filterwarnings('ignore')
 %matplotlib inline
import pandas as pd
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import keras
import operator<load_from_csv>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
df_train=pd.read_csv('.. /input/train.csv')
 df_test=pd.read_csv('.. /input/test.csv' )<count_values>
def detect_outliers(df, n, features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col], 75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(df_train, 2, ["Age", "SibSp", "Parch", "Fare"] )
print("Train data target 1")
 print(df_train[df_train['target']==1].count())
 print("Train data target 0")
 print(df_train[df_train['target']==0].count() )<string_transform>
df_train.loc[Outliers_to_drop]
all_phrases=df_train[df_train.target != 2]
all_words = []
for t in all_phrases.question_text:
all_words.append(t)
 all_words[:4]<concatenate>
df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
all_text = pd.Series(all_words ).str.cat(sep=' ' )<set_options>
for col in df_train.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_train[col].isnull().sum() / df_train[col].shape[0]))
print(msg)
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(all_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()<string_transform>
for col in df_test.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_test[col].isnull().sum() / df_test[col].shape[0]))
print(msg )
neg_phrases = df_train[df_train.target == 1]
neg_words = []
for t in neg_phrases.question_text:
neg_words.append(t)
 neg_words[:4]<concatenate>
df_train[['Pclass', 'Survived']].groupby(['Pclass'], as_index = True ).count()
neg_text = pd.Series(neg_words ).str.cat(sep=' ')
 neg_text[:100]<set_options>
df_train["FamilySize"] = df_train["SibSp"] + df_train["Parch"]+1
df_test["FamilySize"] = df_test["SibSp"] + df_test["Parch"]+1
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(neg_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()<string_transform>
df_train["Fare"] = df_train["Fare"].map(lambda i:np.log(i)if i>0 else 0 )
pos_phrases = df_train[df_train.target == 0]
pos_words = []
for t in pos_phrases.question_text:
pos_words.append(t)
 pos_words[:4]<concatenate>
df_train["Age"].isnull().sum()
pos_text = pd.Series(pos_words ).str.cat(sep=' ')
 pos_text[:100]<set_options>
df_train["Initial"] = df_train["Name"].str.extract("([A-Za-z]+)\.")
 df_test["Initial"] = df_test["Name"].str.extract("([A-Za-z]+)\." )
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(pos_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()<feature_engineering>
df_train["Initial"].replace(["Mlle","Mme", "Ms", "Dr","Major","Lady","Countess", "Jonkheer", "Col", "Rev", "Capt", "Sir", "Don", "Dona"],
["Miss", "Miss","Miss", "Mr", "Mr", "Mrs", "Mrs", "Other", "Other", "Other", "Mr", "Mr", "Mr", "Mr"], inplace = True)
 df_test["Initial"].replace(["Mlle","Mme", "Ms", "Dr","Major","Lady","Countess", "Jonkheer", "Col", "Rev", "Capt", "Sir", "Don", "Dona"],
["Miss", "Miss","Miss", "Mr", "Mr", "Mrs", "Mrs", "Other", "Other", "Other", "Mr", "Mr", "Mr", "Mr"], inplace = True )
df_train['length'] = df_train['question_text'].str.count(' ')+ 1
df_test['length'] = df_test['question_text'].str.count(' ')+ 1<feature_engineering>
df_train.groupby("Initial" ).mean()
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<categorify>
df_all.groupby("Initial" ).mean()
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )<define_variables>
df_train.loc[(df_train["Age"].isnull())&(df_train["Initial"] == "Mr"), "Age"] = 33
df_train.loc[(df_train["Age"].isnull())&(df_train["Initial"] == "Master"),"Age"] = 5
df_train.loc[(df_train["Age"].isnull())&(df_train["Initial"] == "Miss"), "Age"] = 22
df_train.loc[(df_train["Age"].isnull())&(df_train["Initial"] == "Mrs"), "Age"] = 37
df_train.loc[(df_train["Age"].isnull())&(df_train["Initial"] == "Other"), "Age"] = 45
df_test.loc[(df_test["Age"].isnull())&(df_test["Initial"] == "Mr"), "Age"] = 33
df_test.loc[(df_test["Age"].isnull())&(df_test["Initial"] == "Master"),"Age"] = 5
df_test.loc[(df_test["Age"].isnull())&(df_test["Initial"] == "Miss"), "Age"] = 22
df_test.loc[(df_test["Age"].isnull())&(df_test["Initial"] == "Mrs"), "Age"] = 37
df_test.loc[(df_test["Age"].isnull())&(df_test["Initial"] == "Other"), "Age"] = 45
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {'examinaton': 'examination',
'undergraduation': 'under graduation',
'fiancé': 'fiance',
'qoura': 'quora',
'bhakts': 'followers',
'quorans': 'quora users',
'brexit': 'Britain exit',
'cryptocurrencies': 'cryptocurrency',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'pokémon': 'pokemon',
'paralizing': 'paralising',
'perfeccionism': 'perfectionism',
'depreciaton': 'depreciation',
'abvicable': 'abdicable',
'catanation': 'catenation',
'leasership': 'leadership',
'webassembly': 'web assembly',
'fortitide': 'fortitude',
'withdrow': 'withdraw',
'bomblasts': 'bomb blasts',
'engineerer': 'engineer',
'citycarclean': 'city car clean',
'billionsites': 'billion sites',
'willhandjob': 'will hand job',
'fireguns': 'fire guns',
'justeat': 'just eat',
'ubereats': 'uber eats',
'doinformation': 'do information',
'freshersworld': 'freshers world',
'topicwise': 'topic wise',
'excitee': 'excited',
'bengalore': 'bangalore',
'proproetor': 'proprietor',
'migeration': 'migration',
'ejectulate': 'Ejaculate',
'glucoze': 'glucose',
'whatapp': 'whatsapp',
'sumup': 'sum up',
'besic': 'basic',
'experienceed': 'experienced',
'feminisam': 'feminism',
'kayboard': 'keyboard',
'retructuring': 'restructuring',
'becomd': 'become',
'preidct': 'predict',
'statups': 'startups',
'superbrains': 'super brains',
'becoome': 'become',
'gwroth': 'growth',
'wakeupnow': 'wake up now',
'headpone': 'headphone',
'industiry': 'industry',
'arichtecture': 'architecture',
'simlarity': 'similarity',
'walmartlabs': 'walmart labs',
'thunderstike': 'thunder stike',
'maintanable': 'maintainable',
'diffferently': 'differently',
'careamics': 'ceramics',
'sinnister': 'sinister',
'quoras': 'quora',
'breakimg': 'breaking',
'surggery': 'surgery',
'whatwill': 'what will',
'adhaar': 'identity',
'aidentity': 'identity',
'upwork': 'up work',
'alshamsi': 'al shamsi',
'litecoin': 'cryptocurrency ',
'chapterwise': 'chapter wise',
'blockchains': 'blockchain',
'flipcart': 'flipkart',
'Terroristan': 'terrorist Pakistan',
'terroristan': 'terrorist Pakistan',
'BIMARU': 'Bihar, Madhya Pradesh, Rajasthan, Uttar Pradesh',
'Hinduphobic': 'Hindu phobic',
'hinduphobic': 'Hindu phobic',
'Hinduphobia': 'Hindu phobic',
'hinduphobia': 'Hindu phobic',
'Babchenko': 'Arkady Arkadyevich Babchenko faked death',
'Boshniaks': 'Bosniaks',
'Dravidanadu': 'Dravida Nadu',
'mysoginists': 'misogynists',
'MGTOWS': 'Men Going Their Own Way',
'mongloid': 'Mongoloid',
'unsincere': 'insincere',
'meninism': 'male feminism',
'jewplicate': 'jewish replicate',
'unoin': 'Union',
'daesh': 'Islamic State of Iraq and the Levant',
'Kalergi': 'Coudenhove-Kalergi',
'Bhakts': 'Bhakt',
'bhakts': 'Bhakt',
'Tambrahms': 'Tamil Brahmin',
'Pahul': 'Amrit Sanskar',
'SJW': 'social justice warrior',
'SJWs': 'social justice warrior',
'incel': ' involuntary celibates',
'incels': ' involuntary celibates',
'emiratis': 'Emiratis',
'weatern': 'western',
'westernise': 'westernize',
'Pizzagate': 'Pizzagate conspiracy theory',
'naïve': 'naive',
'Skripal': 'Sergei Skripal',
'Remainers': 'British remainer',
'remainers': 'British remainer',
'bremainer': 'British remainer',
'antibrahmin': 'anti Brahminism',
'HYPSM': 'Harvard, Yale, Princeton, Stanford, MIT',
'HYPS': 'Harvard, Yale, Princeton, Stanford',
'kompromat': 'compromising material',
'Tharki': 'pervert',
'tharki': 'pervert',
'mastuburate': 'masturbate',
'Zoë': 'Zoe',
'indans': 'Indian',
'xender': 'gender',
'Naxali ': 'Naxalite ',
'Naxalities': 'Naxalites',
'Bathla': 'Namit Bathla',
'Mewani': 'Indian politician Jignesh Mevani',
'clichéd': 'cliche',
'cliché': 'cliche',
'clichés': 'cliche',
'Wjy': 'Why',
'Fadnavis': 'Indian politician Devendra Fadnavis',
'Awadesh': 'Indian engineer Awdhesh Singh',
'Awdhesh': 'Indian engineer Awdhesh Singh',
'Khalistanis': 'Sikh separatist movement',
'madheshi': 'Madheshi',
'BNBR': 'Be Nice, Be Respectful',
'Bolsonaro': 'Jair Bolsonaro',
'XXXTentacion': 'Tentacion',
'Padmavat': 'Indian Movie Padmaavat',
'Žižek': 'Slovenian philosopher Slavoj Žižek',
'Adityanath': 'Indian monk Yogi Adityanath',
'Brexit': 'British Exit',
'Brexiter': 'British Exit supporter',
'Brexiters': 'British Exit supporters',
'Brexiteer': 'British Exit supporter',
'Brexiteers': 'British Exit supporters',
'Brexiting': 'British Exit',
'Brexitosis': 'British Exit disorder',
'brexit': 'British Exit',
'brexiters': 'British Exit supporters',
'jallikattu': 'Jallikattu',
'fortnite': 'Fortnite ',
'Swachh': 'Swachh Bharat mission campaign ',
'Quorans': 'Quoran',
'Qoura ': 'Quora ',
'quoras': 'Quora',
'Quroa': 'Quora',
'QUORA': 'Quora',
'narcissit': 'narcissist',
'Doklam': 'Tibet',
'Drumpf': 'Donald Trump fool',
'Drumpfs': 'Donald Trump fools',
'Strzok': 'Hillary Clinton scandal',
'rohingya': 'Rohingya ',
'wumao': 'cheap Chinese stuff',
'wumaos': 'cheap Chinese stuff',
'Sanghis': 'Sanghi',
'Tamilans': 'Tamils',
'biharis': 'Biharis',
'Rejuvalex': 'hair growth formula',
'Feku': 'Fake',
'deplorables': 'deplorable',
'muhajirs': 'Muslim immigrant',
'Gujratis': 'Gujarati',
'Chutiya': 'Fucker',
'Chutiyas': 'Fucker',
'thighing': 'masturbate',
'卐': 'Nazi Germany',
'Pribumi': 'Native Indonesian',
'Gurmehar': 'Gurmehar Kaur Indian student activist',
'Novichok': 'Soviet Union agents',
'Khazari': 'Khazars',
'Demonetization': 'demonetization',
'demonetisation': 'demonetization',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'cryptocurrencies': 'cryptocurrency',
'Hindians': 'North Indian who hate British',
'vaxxer': 'vocal nationalist ',
'remoaner': 'remainer ',
'bremoaner': 'British remainer ',
'Jewism': 'Judaism',
'Eroupian': 'European',
'WMAF': 'White male married Asian female',
'moeslim': 'Muslim',
'cishet': 'cisgender and heterosexual person',
'Eurocentric': 'Eurocentrism ',
'Jewdar': 'Jew dar',
'Asifa': 'abduction, rape, murder case ',
'marathis': 'Marathi',
'Trumpanzees': 'Trump chimpanzee fool',
'Crimean': 'Crimea people ',
'atrracted': 'attract',
'LGBT': 'lesbian, gay, bisexual, transgender',
'Boshniak': 'Bosniaks ',
'Myeshia': 'widow of Green Beret killed in Niger',
'demcoratic': 'Democratic',
'raaping': 'rape',
'Dönmeh': 'Islam',
'feminazism': 'feminism nazi',
'langague': 'language',
'Hongkongese': 'HongKong people',
'hongkongese': 'HongKong people',
'Kashmirians': 'Kashmirian',
'Chodu': 'fucker',
'penish': 'penis',
'micropenis': 'tiny penis',
'Madridiots': 'Real Madrid idiot supporters',
'Ambedkarite': 'Dalit Buddhist movement ',
'ReleaseTheMemo': 'cry for the right and Trump supporters',
'harrase': 'harass',
'Barracoon': 'Black slave',
'Castrater': 'castration',
'castrater': 'castration',
'Rapistan': 'Pakistan rapist',
'rapistan': 'Pakistan rapist',
'Turkified': 'Turkification',
'turkified': 'Turkification',
'Dumbassistan': 'dumb ass Pakistan',
'facetards': 'Facebook retards',
'rapefugees': 'rapist refugee',
'superficious': 'superficial',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'narcissit': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
'mastrubating': 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
'whst': 'what',
'watsapp': 'whatsapp',
'bodyshame': 'body shaming',
'bodyshoppers': 'body shopping',
'bodycams': 'body cams',
'Cananybody': 'Can any body',
'deadbody': 'dead body',
'deaddict': 'de addict',
'Northindian': 'North Indian ',
'northindian': 'north Indian ',
'northkorea': 'North Korea',
'Whykorean': 'Why Korean',
'koreaboo': 'Korea boo ',
'Brexshit': 'British Exit bullshit',
'shithole': 'shithole ',
'shitpost': 'shit post',
'shitslam': 'shit Islam',
'shitlords': 'shit lords',
'Fck': 'Fuck',
'fck': 'fuck',
'Clickbait': 'click bait ',
'clickbait': 'click bait ',
'mailbait': 'mail bait',
'healhtcare': 'healthcare',
'trollbots': 'troll bots',
'trollled': 'trolled',
'trollimg': 'trolling',
'cybertrolling': 'cyber trolling',
'sickular': 'India sick secular ',
'suckimg': 'sucking',
'Idiotism': 'idiotism',
'Niggerism': 'Nigger',
'Niggeriah': 'Nigger'} 
<load_pretrained>
df_train["Age"].isnull().sum()
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index 
<load_pretrained>
df_test["Age"].isnull().sum()
glove_embeddings = load_embed(glove)
 print(len(glove_embeddings))<concatenate>
df_train["Embarked"].isnull().sum()
train = df_train['question_text']
test = df_test['question_text']
df = pd.concat([train ,test])
 vocab = build_vocab(df )<compute_test_metric>
df_train["Embarked"].fillna("S", inplace = True)
print("oov : Glove ")
 oov = check_coverage(vocab, glove_embeddings)
 add_lower(glove_embeddings, vocab)
 print("oov : ")
 oov = check_coverage(vocab, glove_embeddings)
<string_transform>
df_test["Embarked"].isnull().sum()
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text<categorify>
df_train["Age_Categ"] = 0
df_test["Age_Categ"] = 0
def clean_special_chars(text, punct, puncts, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 for p in puncts:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text<categorify>
def category_age(x):
if x < 10:
return 0
elif x < 20:
return 1
elif x < 30:
return 2
elif x < 40:
return 3
elif x < 50:
return 4
elif x < 60:
return 5
elif x < 70:
return 6
else:
return 7
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<categorify>
df_train["Age_Categ"] = df_train["Age"].apply(category_age)
 df_test["Age_Categ"] = df_test["Age"].apply(category_age)
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<feature_engineering>
df_train.drop(["Age"], axis = 1 ,inplace = True)
 df_test.drop(["Age"], axis = 1, inplace = True )
df_train['question_text'] = df_train['question_text'].apply(lambda x: x.lower())
 df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, puncts, punct_mapping))
df_train['question_text'] = df_train['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_numbers(x))<feature_engineering>
df_train["Initial"] = df_train["Initial"].map({"Master" : 0, "Miss" : 1, "Mr" : 2, "Mrs" : 3, "Other" : 4})
 df_test["Initial"] = df_test["Initial"].map({"Master" : 0, "Miss" : 1, "Mr" : 2, "Mrs" : 3, "Other" : 4} )
df_test['question_text'] = df_test['question_text'].apply(lambda x: x.lower())
 df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct,puncts, punct_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_numbers(x))<concatenate>
df_train["Embarked"].value_counts()
train = df_train['question_text']
test = df_test['question_text']
df = pd.concat([train ,test])
 vocab = build_vocab(df)
 print("oov : ")
 oov = check_coverage(vocab, glove_embeddings )<split>
df_train["Embarked"] = df_train["Embarked"].map({"C" : 0, "Q" : 1, "S" : 2})
 df_test["Embarked"] = df_test["Embarked"].map({"C" : 0, "Q" : 1, "S" : 2} )
train_df, test2_df = train_test_split(df_train, test_size=0.04, random_state=123)
 train_df, valid_df = train_test_split(train_df, test_size=0.00001, random_state=123)
 train_df=train_df.reset_index(drop=True)
 valid_df=valid_df.reset_index(drop=True)
 test2_df=test2_df.reset_index(drop=True)
 X_train=train_df['question_text'].values
y_train=train_df['target'].values
X_test2=test2_df['question_text'].values
y_test2=test2_df['target'].values
X_valid=valid_df['question_text'].values
y_valid=valid_df['target'].values
X_test=df_test['question_text'].values
print(X_train.shape)
 print(y_train.shape)
 print(X_valid.shape)
 print(y_valid.shape)
 print(X_test2.shape)
 print(y_test2.shape)
 print(X_test.shape)
 print(train_df.shape)
 print(valid_df.shape)
 print(test2_df.shape )<define_variables>
df_train["Sex"] = df_train["Sex"].map({"female" : 0, "male" : 1})
 df_test["Sex"] = df_test["Sex"].map({"female" : 0, "male" : 1} )
embedding_dim = 300
max_features = 120000
maxlen = 70
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(X_train.tolist() + X_valid.tolist() + X_test2.tolist() + X_test.tolist() )<define_variables>
heatmap_data = df_train[["Survived", "Pclass", "Sex", "Fare", "Embarked", "FamilySize", "Initial", "Age_Categ"]]
vocab_size = len(tokenizer.word_index)+ 1
print(vocab_size )<feature_engineering>
df_train = pd.get_dummies(df_train, columns = ["Initial"], prefix = "Initial")
 df_test = pd.get_dummies(df_test, columns = ["Initial"], prefix = "Initial" )
def index_to_matrix(embeddings_index,word_index):
embedding_matrix = np.zeros(( len(word_index)+ 1, embedding_dim))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return(embedding_matrix )<prepare_x_and_y>
df_train = pd.get_dummies(df_train, columns = ["Embarked"], prefix = "Embarked")
 df_test = pd.get_dummies(df_test, columns = ["Embarked"], prefix = "Embarked" )
glove_embedding_matrix=index_to_matrix(glove_embeddings,tokenizer.word_index)
 embedding_matrix=glove_embedding_matrix<import_modules>
df_train.drop(["PassengerId", "Name", "SibSp", "Parch", "Ticket", "Cabin"], axis = 1, inplace = True)
 df_test.drop(["PassengerId", "Name", "SibSp", "Parch", "Ticket", "Cabin"], axis = 1, inplace = True )
from keras.models import Model
from keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU,CuDNNLSTM, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, Add
from keras.optimizers import Adam
from keras.models import Sequential
from keras import layers
import keras.callbacks
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint<choose_model_class>
kfold = StratifiedKFold(n_splits=10 )
def make_model(embedding_matrix, maxlen, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(maxlen,))
inp2 = Input(shape=(1,))
x = Embedding(vocab_size, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Dropout(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Dropout(0.2 )(x)
 avg_pl = GlobalAveragePooling1D()(x)
 max_pl = GlobalMaxPooling1D()(x)
 concat = concatenate([avg_pl, max_pl])
 dense1 = Dense(32, activation="relu" )(concat)
 concat = concatenate([dense1, inp2])
 output = Dense(1, activation="sigmoid" )(concat)
 model = Model(inputs=[inp,inp2], outputs=output)
 model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])
 return model<choose_model_class>
df_train["Survived"] = df_train["Survived"].astype(int)
 Y_train = df_train["Survived"]
X_train = df_train.drop(labels = ["Survived"],axis = 1 )
model = make_model(embedding_matrix,maxlen=70 )<choose_model_class>
random_state = 2
classifiers = []
classifiers.append(SVC(random_state = random_state))
classifiers.append(DecisionTreeClassifier(random_state = random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state = random_state), random_state = random_state, learning_rate = 0.1))
classifiers.append(RandomForestClassifier(random_state = random_state))
classifiers.append(ExtraTreesClassifier(random_state = random_state))
classifiers.append(GradientBoostingClassifier(random_state = random_state))
classifiers.append(MLPClassifier(random_state = random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(LinearDiscriminantAnalysis())
 cv_results = []
for classifier in classifiers:
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs = 4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans": cv_means, "CrossValerrors": cv_std,
"Algorithm": ["SVC", "DecisionTree", "AdaBoost", "RandomForest",
"ExtraTrees", "GradientBoosting", "MultipleLayerPerceptron", "KNeighboors",
"LogisticRegression", "LinearDiscriminantAnalysis"]})
 g = sns.barplot("CrossValMeans", "Algorithm", data = cv_res, palette = "Set3",
orient = "h", **{'xerr': cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )
seed = 7
n_splits=5
np.random.seed(seed)
 kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed )<split>
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state = 7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter": ["best", "random"],
"algorithm": ["SAMME", "SAMME.R"],
"n_estimators": [1,2],
"learning_rate": [0.0001,0.001, 0.01, 0.1, 0.2, 0.3, 1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv = kfold, scoring = "accuracy",
n_jobs = 4, verbose = 1)
 gsadaDTC.fit(X_train, Y_train)
 ada_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_
i=1
maxlength={}
maxl={}
for train, valid in kfold.split(X_train, y_train):
if i <=n_splits:
maxl[i]=X_len_train[train].max()
print(X_len_train[train])
 maxlength[i]=int(np.quantile(X_len_train[train],0.999))
print("Running Fold", i, "/", n_splits)
 print("split 99.9 percentile length",maxlength[i],"split max length",maxl[i])
 modelname=str("Model")+ str(i)
 train_data=[X_train[j] for j in train]
valid_data=[X_train[k] for k in valid]
train_data = tokenizer.texts_to_sequences(train_data)
 valid_data = tokenizer.texts_to_sequences(valid_data)
 train_data = pad_sequences(train_data, padding='post', maxlen=maxlength[i])
 valid_data = pad_sequences(valid_data, padding='post', maxlen=maxlength[i])
 model = make_model(embedding_matrix,maxlength[i])
 print("train", y_train[train].sum())
 print("validation",y_train[valid].sum())
 checkpointer = ModelCheckpoint(filepath=modelname,monitor='val_loss', mode='auto', verbose = 1, save_best_only=True)
 history = model.fit([train_data,X_len_train[train]], y_train[train],
epochs=5,
validation_data=([valid_data,X_len_train[valid]], y_train[valid]),
batch_size=512,callbacks=[checkpointer])
 plot_history(history)
 i=i+1<predict_on_test>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1,2,10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1,3,10],
"bootstrap": [False],
"n_estimators": [100, 300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC, param_grid = ex_param_grid, cv = kfold, scoring = "accuracy",
n_jobs = 4, verbose = 1)
 gsExtC.fit(X_train, Y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_
y_pred={}
y_pred_test={}
count=0
for i in np.arange(1, n_splits+1, 1):
try:
model.load_weights(str("Model")+ str(i))
print(str("Model")+ str(i))
x_test2 = tokenizer.texts_to_sequences(X_test2)
 x_test = tokenizer.texts_to_sequences(X_test)
 x_test2 = pad_sequences(x_test2, padding='post', maxlen=maxlength[i])
 x_test = pad_sequences(x_test, padding='post', maxlen=maxlength[i])
 y_pred[i] = model.predict([x_test2,X_len_test2], batch_size=512, verbose=1)
 y_pred_test[i] = model.predict([x_test,X_len_test], batch_size=512, verbose=1)
 model_f1_score={}
for thresh in np.arange(0.1, 0.91, 0.01):
thresh = np.round(thresh, 2)
 model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2,(y_pred[i]>=thresh ).astype(int))
model_cutoff=max(model_f1_score, key=model_f1_score.get)
 print("Max F1 score is {1} found at threshold {0}".format(model_cutoff, model_f1_score[model_cutoff]))
count=count+1
except:
pass
print('count is ', count)
<define_variables>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1,3,10],
"min_samples_split": [2,3,10],
"min_samples_leaf": [1,2,10],
"bootstrap": [False],
"n_estimators": [100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC, param_grid = rf_param_grid, cv=kfold, scoring = "accuracy", n_jobs = 4,
verbose = 1)
 gsRFC.fit(X_train,Y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_
y_pred_final={}
y_pred_test_final={}
for i in np.arange(1, count+1, 1):
if(i == 1):
y_pred_final=y_pred[i]
y_pred_test_final=y_pred_test[i]
else:
y_pred_final=y_pred_final + y_pred[i]
y_pred_test_final=y_pred_test_final + y_pred_test[i]
y_pred_final=y_pred_final/count
y_pred_test_final=y_pred_test_final/count 
<find_best_params>
GBC = GradientBoostingClassifier()
gb_param_grid = {"loss": ["deviance"],
"n_estimators": [100,200,300],
"learning_rate": [0.1, 0.05, 0.01],
"max_depth": [4, 8],
"min_samples_leaf": [100,150],
"max_features": [0.3, 0.1]}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv = kfold, scoring = "accuracy",
n_jobs = 4, verbose = 1)
 gsGBC.fit(X_train, Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_
print("Final Model on hold out")
 model_f1_score={}
for thresh in np.arange(0.1, 0.91, 0.01):
thresh = np.round(thresh, 2)
 model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2,(y_pred_final>=thresh ).astype(int))
model_cutoff=max(model_f1_score, key=model_f1_score.get)
 print("Max F1 score is {1} found at threshold {0}".format(model_cutoff, model_f1_score[model_cutoff]))<compute_test_metric>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_
y_pred_test2_final_class =(y_pred_final >= model_cutoff ).astype(int)
 print(y_test2.sum())
 print(y_pred_test2_final_class.sum())
 print(sklearn.metrics.f1_score(y_test2, y_pred_test2_final_class))
print('classification report')
 print(classification_report(y_test2, y_pred_test2_final_class))
print('Confusion matrix')
 print(sklearn.metrics.confusion_matrix(y_test2, y_pred_test2_final_class))
print('roc score')
 print(sklearn.metrics.roc_auc_score(y_test2, y_pred_test2_final_class))<create_dataframe>
votingC = VotingClassifier(estimators = [("rfc", RFC_best),("extc", ExtC_best), 
("svc", SVMC_best),("adac", ada_best), 
("gbc", GBC_best)], voting = "soft", n_jobs = 4)
 votingC = votingC.fit(X_train, Y_train )
df=pd.DataFrame(columns=['text','y_actual', 'y_pred','y_pred_prob','length'] )<feature_engineering>
submission = pd.read_csv(".. /input/gender_submission.csv" )
df['text'] = X_test2
df['y_pred'] =y_pred_test2_final_class
df['y_pred_prob'] =y_pred_final
df['length'] =X_len_test2
df['y_actual'] =y_test2<filter>
df_test["Fare"].fillna("35.6271", inplace = True)
 X_test = df_test.values
df[(df.y_actual != df.y_pred)&(df.y_pred_prob >=(model_cutoff-0.3)) &(df.y_pred_prob <=(model_cutoff + 0.3)) ]['text'].values<set_options>
prediction = votingC.predict(X_test )
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(FN_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()<count_values>
submission["Survived"] = prediction
counts = Counter(FN_text.split())
 print(counts )<count_values>
submission.to_csv("./The_first_submission.csv", index = False )