Kaggle competition: Titanic - Machine Learning from Disaster
from sklearn.utils import shuffle<load_from_csv>
data=pd.read_csv('.. /input/titanic/train.csv')
 data
df = pd.read_csv('.. /input/train.csv' )<load_from_csv>
data['Ticket_type']=data['Ticket'].apply(lambda x: x[0:3])
 data['Ticket_type']=data['Ticket_type'].astype('category' ).cat.codes
test_df = pd.read_csv(".. /input/test.csv" )<count_unique_values>
data['Words_counts']=data['Name'].apply(lambda x: len(x.split()))
len(df.target.unique() )<rename_columns>
data['cabin_or_not']=data["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 data.head(3 )
df.columns = ['qid', 'text', 'category']
test_df.columns = ['qid', 'text']<drop_column>
data['Family_size']=data['SibSp'] + data['Parch'] + 1
df = df[['text', 'category']]<feature_engineering>
data['IsAlone'] = 0
data.loc[data['Family_size'] == 1, 'IsAlone'] = 1
data.head(3 )
df['text'] = df['text'].str.lower()
test_df['text'] = test_df['text'].str.lower()<define_variables>
data['Embarked'] = data['Embarked'].fillna('S')
 data['Age'].fillna(data['Age'].mean() ,inplace=True )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'u.s':'america', 'e.g':'for example'}<string_transform>
data['fare_cat']=pd.qcut(data['Fare'], 4)
 data['fare_cat']=data['fare_cat'].astype('category' ).cat.codes.astype('int' )
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text<feature_engineering>
data['cat_age']=pd.cut(data['Age'],5)
 print(data['cat_age'].value_counts())
 data['cat_age']=data['cat_age'].astype('category' ).cat.codes.astype('int' )
df['text'] = df['text'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['text'] = test_df['text'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))<define_variables>
title=title.replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'],'Rare')
 title=title.replace('Mlle','Miss')
 title=title.replace('Ms','Miss')
 title=title.replace('Mme','Mrs')
 title.value_counts()
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]<define_variables>
data['Title']=title
dic1= {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
data['Title'] = data['Title'].map(dic1)
 data['Title'] =data['Title'].fillna(0 )
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!':' '}<categorify>
data['Sex'] = data['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 data.head(3 )
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text<feature_engineering>
data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
df['text'] = df['text'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['text'] = test_df['text'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))<define_variables>
data.drop(columns=['PassengerId','Name','Ticket','Cabin'],inplace=True )
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}<categorify>
train=pd.read_csv('.. /input/titanic/train.csv')
 test=pd.read_csv('.. /input/titanic/test.csv' )
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<feature_engineering>
test['Fare'].fillna(test['Fare'].mean() ,inplace=True )
df['text'] = df['text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df['text'] = test_df['text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))<define_variables>
datas=[train,test]
for data in datas:
data['Ticket_type']=data['Ticket'].apply(lambda x: x[0:3])
 data['Ticket_type']=data['Ticket_type'].astype('category' ).cat.codes
data['Words_counts']=data['Name'].apply(lambda x: len(x.split()))
data['cabin_or_not']=data["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 data['Family_size']=data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 0
data.loc[data['Family_size'] == 1, 'IsAlone'] = 1
data['Embarked'] = data['Embarked'].fillna('S')
 data['fare_cat']=pd.qcut(data['Fare'], 4)
 data['Age'].fillna(data['Age'].mean() ,inplace=True)
 data['cat_age']=pd.cut(data['Age'],5)
 title=[]
for i in range(0,len(data)) :
title.append(re.search('([A-Za-z]+)\.', data['Name'][i] ).group(1))
title=pd.Series(title)
 title=title.replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'],'Rare')
 title=title.replace('Mlle','Miss')
 title=title.replace('Ms','Miss')
 title=title.replace('Mme','Mrs')
 data['Title']=title
data['Sex'] = data['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 dic1= {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
data['Title'] = data['Title'].map(dic1)
 data['Title'] =data['Title'].fillna(0)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype('int')
 data['fare_cat']=data['fare_cat'].astype('category' ).cat.codes.astype('int')
 data['cat_age']=data['cat_age'].astype('category' ).cat.codes.astype('int')
 data.drop(columns=['PassengerId','Name','Ticket','Cabin'],inplace=True)
labels = df['category']
text = df['text']<prepare_x_and_y>
x=train.drop(columns=['Survived'])
 y=train['Survived']
train_text = text.reset_index().drop('index', axis=1)
 y_train = labels.reset_index().drop('index', axis=1)
 val_text = None
y_val = None<drop_column>
log_model = LogisticRegression(C=22 ).fit(x, y )
test = test_df['text']<groupby>
submitssion_dis={'PassengerId':[a for a in range(892,1310)]
,'Survived':log_model.predict(test)}
cates = df.groupby('category')
 print("total categories:", cates.ngroups)
 print(cates.size() )<define_variables>
submission=pd.DataFrame(submitssion_dis)
 submission.to_csv('Submission_out.csv',index=False )