import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline

# Define the status function if it's not already defined
def status(feature_name):
    print(f"Processed feature: {feature_name}")

# 1. Combine Train and Test Data
def get_combined_data(train_path='./data/train.csv', test_path='./data/test.csv'):
    train = pd.read_csv(train_path)
    test = pd.read_csv(test_path)

    targets = train['Survived']
    train = train.drop(['Survived'], axis=1)

    combined = pd.concat([train, test], ignore_index=True)
    combined = combined.drop(['PassengerId'], axis=1)

    return combined, targets

# 2. Extract Titles from Names
def get_titles(combined):
    Title_Dictionary = {
        "Mr": "Mr",
        "Mrs": "Mrs",
        "Miss": "Miss",
        "Master": "Master",
        "Dr": "Officer",
        "Rev": "Officer",
        "Col": "Officer",
        "Major": "Officer",
        "Mlle": "Miss",
        "Countess": "Royalty",
        "Ms": "Miss",
        "Lady": "Royalty",
        "Jonkheer": "Royalty",
        "Don": "Royalty",
        "Dona": "Royalty",
        "Mme": "Mrs",
        "Capt": "Officer",
        "Sir": "Royalty"
    }
    combined['Title'] = combined['Name'].apply(lambda name: name.split(',')[1].split('.')[0].strip())
    combined['Title'] = combined['Title'].map(Title_Dictionary)
    status('Title')
    return combined

# 3. Fill Missing Ages
def process_age(combined):
    # Group median for Age based on Sex, Pclass, and Title
    grouped_median = combined.groupby(['Sex', 'Pclass', 'Title'])['Age'].median().reset_index()

    def fill_age(row):
        condition = (
            (grouped_median['Sex'] == row['Sex']) &
            (grouped_median['Title'] == row['Title']) &
            (grouped_median['Pclass'] == row['Pclass'])
        )
        # Use [0] instead of (0) to access the first element
        age = grouped_median[condition]['Age'].values
        if len(age) > 0:
            return age[0]
        else:
            return combined['Age'].median()  # Fallback in case no match is found

    combined['Age'] = combined.apply(
        lambda row: fill_age(row) if pd.isnull(row['Age']) else row['Age'], axis=1
    )
    status('Age')
    return combined

# 4. Fill Missing Fares
def process_fares(combined):
    # Calculate the mean Fare from the training set (first 891 rows)
    mean_fare = combined.loc[:890, 'Fare'].mean()

    # Replace missing Fare values with the calculated mean
    combined['Fare'] = combined['Fare'].fillna(mean_fare)

    status('Fare')
    return combined

# 5. Encode Sex
def process_sex(combined):
    combined['Sex'] = combined['Sex'].map({'male': 1, 'female': 0})
    status('Sex')
    return combined

# Example Usage:
# Step 1: Combine Data
combined, targets = get_combined_data()

# Step 2: Extract Titles
combined = get_titles(combined)

# Step 3: Process Age
combined = process_age(combined)

# Step 4: Process Fares
combined = process_fares(combined)

# Step 5: Encode Sex
combined = process_sex(combined)

# Continue with other preprocessing steps...



Q- let’s continue so 

Processing Pclass
In [50]:
def process_pclass():
    
    global combined
    # encoding into 3 categories:
    pclass_dummies = pd.get_dummies(combined['Pclass'], prefix="Pclass")
    
    # adding dummy variable
    combined = pd.concat([combined, pclass_dummies],axis=1)
    
    # removing "Pclass"
    combined.drop('Pclass',axis=1,inplace=True)
    
    status('Pclass')
    return combined


This function encodes the values of Pclass (1,2,3) using a dummy encoding.
In [51]:
combined = process_pclass()


Processing Pclass : ok
Processing Ticket
Let's first see how the different ticket prefixes we have in our dataset
In [52]:
def cleanTicket(ticket):
    ticket = ticket.replace('.', '')
    ticket = ticket.replace('/', '')
    ticket = ticket.split()
    ticket = map(lambda t : t.strip(), ticket)
    ticket = list(filter(lambda t : not t.isdigit(), ticket))
    if len(ticket) > 0:
        return ticket[0]
    else: 
        return 'XXX'

In [53]:
tickets = set()
for t in combined['Ticket']:
    tickets.add(cleanTicket(t))

In [54]:
print len(tickets)

37

In [55]:
def process_ticket():
    
    global combined
    
    # a function that extracts each prefix of the ticket, returns 'XXX' if no prefix (i.e the ticket is a digit)
    def cleanTicket(ticket):
        ticket = ticket.replace('.','')
        ticket = ticket.replace('/','')
        ticket = ticket.split()
        ticket = map(lambda t : t.strip(), ticket)
        ticket = filter(lambda t : not t.isdigit(), ticket)
        if len(ticket) > 0:
            return ticket[0]
        else: 
            return 'XXX'
    

    # Extracting dummy variables from tickets:

    combined['Ticket'] = combined['Ticket'].map(cleanTicket)
    tickets_dummies = pd.get_dummies(combined['Ticket'], prefix='Ticket')
    combined = pd.concat([combined, tickets_dummies], axis=1)
    combined.drop('Ticket', inplace=True, axis=1)

    status('Ticket')
    return combined

In [56]:
combined = process_ticket()

Processing Ticket : ok

Processing Family
This part includes creating new variables based on the size of the family (the size is by the way, another variable we create).
This creation of new variables is done under a realistic assumption: Large families are grouped together, hence they are more likely to get rescued than people traveling alone.
In [57]:
def process_family():
    
    global combined
    # introducing a new feature : the size of families (including the passenger)
    combined['FamilySize'] = combined['Parch'] + combined['SibSp'] + 1
    
    # introducing other features based on the family size
    combined['Singleton'] = combined['FamilySize'].map(lambda s: 1 if s == 1 else 0)
    combined['SmallFamily'] = combined['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)
    combined['LargeFamily'] = combined['FamilySize'].map(lambda s: 1 if 5 <= s else 0)
    
    status('family')
    return combined

This function introduces 4 new features:
FamilySize : the total number of relatives including the passenger (him/her)self.
Sigleton : a boolean variable that describes families of size = 1
SmallFamily : a boolean variable that describes families of 2 <= size <= 4
LargeFamily : a boolean variable that describes families of 5 < size
In [58]:
combined = process_family()

Processing family : ok

In [59]:
print combined.shape

(1309, 67)

We end up with a total of 67 features.
In [60]:
combined.head()

Out[60]:



Sex
Age
SibSp
Parch
Fare
Title_Master
Title_Miss
Title_Mr
Title_Mrs
Title_Officer
Title_Royalty
Embarked_C
Embarked_Q
Embarked_S
Cabin_A
Cabin_B
Cabin_C
Cabin_D
Cabin_E
Cabin_F
Cabin_G
Cabin_T
Cabin_U
Pclass_1
Pclass_2
Pclass_3
Ticket_A
Ticket_A4
Ticket_A5
Ticket_AQ3
Ticket_AQ4
Ticket_AS
Ticket_C
Ticket_CA
Ticket_CASOTON
Ticket_FC
Ticket_FCC
Ticket_Fa
Ticket_LINE
Ticket_LP
Ticket_PC
Ticket_PP
Ticket_PPP
Ticket_SC
Ticket_SCA3
Ticket_SCA4
Ticket_SCAH
Ticket_SCOW
Ticket_SCPARIS
Ticket_SCParis
Ticket_SOC
Ticket_SOP
Ticket_SOPP
Ticket_SOTONO2
Ticket_SOTONOQ
Ticket_SP
Ticket_STONO
Ticket_STONO2
Ticket_STONOQ
Ticket_SWPP
Ticket_WC
Ticket_WEP
Ticket_XXX
FamilySize
Singleton
SmallFamily
LargeFamily
0
1
22.0
1
0
7.2500
0
0
1
0
0
0
0
0
1
0
0
0
0
0
0
0
0
1
0
0
1
0
0
1
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
2
0
1
0
1
0
38.0
1
0
71.2833
0
0
0
1
0
0
1
0
0
0
0
1
0
0
0
0
0
0
1
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
1
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
2
0
1
0
2
0
26.0
0
0
7.9250
0
1
0
0
0
0
0
0
1
0
0
0
0
0
0
0
0
1
0
0
1
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
1
0
0
0
0
0
1
1
0
0
3
0
35.0
1
0
53.1000
0
0
0
1
0
0
0
0
1
0
0
1
0
0
0
0
0
0
1
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
1
2
0
1
0
4
1
35.0
0
0
8.0500
0
0
1

























































































































Q- now my target is to create a file called 

Feature_engineering.py 
It can contain all feature engineering here 

With clean code standard oky