Kaggle competition: IESB - 2019
column_choice_training['Age_is_Null'] = column_choice_training['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 column_choice_test['Age_is_Null'] = column_choice_test['Age'].apply(lambda x: 1 if pd.isnull(x)else 0 )<groupby>
test = pd.read_csv('.. /input/test.csv',low_memory=False)
 df = pd.read_csv('.. /input/train.csv',low_memory=False )
column_choice_training[['Age', 'Pclass', 'Sex']].groupby(['Pclass', 'Sex'], as_index=False ).mean()<categorify>
test['nota_mat'] = -2
age_means = column_choice_training.groupby(['Sex', 'Pclass'])['Age']
column_choice_training.Age = age_means.transform(lambda x: x.fillna(x.mean()))
column_choice_test.Age = age_means.transform(lambda x: x.fillna(x.mean()))
column_choice_training.head(6 )<categorify>
df = df.append(test )
bins = [0, 6, 60, 80]
column_choice_training['Age_cut'] = pd.cut(column_choice_training['Age'], bins)
 column_choice_test['Age_cut'] = pd.cut(column_choice_test['Age'], bins)
 column_choice_training = pd.concat([column_choice_training.drop(['Age_cut'], axis=1),
pd.get_dummies(column_choice_training['Age_cut'], prefix = 'Age_cut')], axis=1)
 column_choice_training = column_choice_training.drop(['Age_cut_(6, 60]'], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Age_cut'], axis=1),
pd.get_dummies(column_choice_test['Age_cut'], prefix = 'Age_cut')], axis=1)
 column_choice_test = column_choice_test.drop(['Age_cut_(6, 60]'], axis=1)
 column_choice_training.head(6 )<data_type_conversions>
df.isnull().sum()
column_choice_training.Fare = column_choice_training.Fare.fillna(column_choice_training.Fare.mean())
 column_choice_test.Fare = column_choice_test.Fare.fillna(column_choice_training.Fare.mean())
 column_choice_training.head(6 )<feature_engineering>
df['codigo_mun'] = df['codigo_mun'].apply(lambda x: x.replace('ID_ID_', ''))
column_choice_training['Name_Length'] = column_choice_training['Name'].apply(len)
 column_choice_test['Name_Length'] = column_choice_test['Name'].apply(len)
 column_choice_training.head(6 )<feature_engineering>
for col in df.columns:
if df[col].dtype == 'object' and col != 'codigo_mun':
df[col] = df[col].astype('category' ).cat.codes
column_choice_training['Title'] = column_choice_training['Name'].str.split(', ' ).str[1]
column_choice_training['Title'] = column_choice_training['Title'].str.split('.' ).str[0]
column_choice_training = column_choice_training.drop(['Name'], axis=1)
 column_choice_test['Title'] = column_choice_test['Name'].str.split(', ' ).str[1]
column_choice_test['Title'] = column_choice_test['Title'].str.split('.' ).str[0]
column_choice_test = column_choice_test.drop(['Name'], axis=1 )<feature_engineering>
removed_cols = ['Unnamed: 0', 'codigo_mun','municipio', 'nota_mat']
feats = [c for c in df.columns if c not in removed_cols]
column_choice_training['Title'] = column_choice_training['Title'].replace(['the Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Rare')
 column_choice_training['Title'] = column_choice_training['Title'].replace(['Lady','Mlle', 'Ms'], 'Mrs')
 column_choice_training['Title'] = column_choice_training['Title'].replace(['Mme'], 'Miss')
 column_choice_test['Title'] = column_choice_test['Title'].replace(['Col', 'Dona', 'Dr', 'Rev'], 'Rare')
 column_choice_test['Title'] = column_choice_test['Title'].replace(['Ms'], 'Mrs')
 pd.crosstab(column_choice_training['Title'], column_choice_training['Sex'] )<feature_engineering>
test = df[df['nota_mat'] == -2]
df = df[~(df['nota_mat'] == -2)]
column_choice_training.Title[column_choice_training.Title == 'Master'] = 0
column_choice_training.Title[column_choice_training.Title == 'Miss'] = 1
column_choice_training.Title[column_choice_training.Title == 'Mr'] = 2
column_choice_training.Title[column_choice_training.Title == 'Mrs'] = 3
column_choice_training.Title[column_choice_training.Title == 'Rare'] = 4
column_choice_test.Title[column_choice_test.Title == 'Master'] = 0
column_choice_test.Title[column_choice_test.Title == 'Miss'] = 1
column_choice_test.Title[column_choice_test.Title == 'Mr'] = 2
column_choice_test.Title[column_choice_test.Title == 'Mrs'] = 3
column_choice_test.Title[column_choice_test.Title == 'Rare'] = 4
column_choice_training.head(6 )<sort_values>
train, valid = train_test_split(df, test_size=0.3333, random_state=42 )
column_choice_training['Ticket_Len'] = column_choice_training['Ticket'].apply(len)
 column_choice_test['Ticket_Len'] = column_choice_test['Ticket'].apply(len)
 print(column_choice_test['Ticket_Len'].value_counts())
 column_choice_training[['Ticket_Len', 'Survived']].groupby(['Ticket_Len'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
rf = RandomForestRegressor(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )
column_choice_training['Ticket_Letter'] = column_choice_training['Ticket'].str[0]
column_choice_test['Ticket_Letter'] = column_choice_test['Ticket'].str[0]
print(column_choice_training['Ticket_Letter'].value_counts())
 print(column_choice_test['Ticket_Letter'].value_counts())
 column_choice_training[['Ticket_Letter', 'Survived']].groupby(['Ticket_Letter'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
rf.fit(train[feats], train['nota_mat'] )
column_choice_training['Ticket_Letter'] = column_choice_training['Ticket_Letter'].replace(['W', '4', '7', '6', 'L', '5', '8'], 'Rare_Low_Surv')
 column_choice_training['Ticket_Letter'] = column_choice_training['Ticket_Letter'].replace(['F', '9'], 'Rare_High_Surv')
 column_choice_test['Ticket_Letter'] = column_choice_test['Ticket_Letter'].replace(['W', '4', '7', '6', 'L', '5', '8'], 'Rare_Low_Surv')
 column_choice_test['Ticket_Letter'] = column_choice_test['Ticket_Letter'].replace(['F', '9'], 'Rare_High_Surv')
 column_choice_training.tail()<categorify>
valid_preds = rf.predict(valid[feats])
 mean_squared_error(( valid['nota_mat']), valid_preds)**(1/2 )
column_choice_training = pd.concat([column_choice_training.drop(['Ticket', 'Ticket_Letter'], axis=1),
pd.get_dummies(column_choice_training['Ticket_Letter'], prefix = 'Ticket_Letter')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Ticket', 'Ticket_Letter'], axis=1),
pd.get_dummies(column_choice_test['Ticket_Letter'], prefix = 'Ticket_Letter')], axis=1)
 column_choice_training.head()<count_values>
removed_cols = ['exp_anos_estudo', 'densidade_dem','gasto_pc_saude', 'hab_p_medico','servidores',
'area','comissionados','comissionados_por_servidor','indice_governanca',
'ranking_igm','porte','capital']
feats = [c for c in df.columns if c not in removed_cols]
column_choice_training.Cabin = column_choice_training.Cabin.fillna('N0')
 column_choice_test.Cabin = column_choice_test.Cabin.fillna('N0')
 column_choice_training['Cabin_Letter'] = column_choice_training['Cabin'].str[0]
column_choice_test['Cabin_Letter'] = column_choice_test['Cabin'].str[0]
print(column_choice_training['Cabin_Letter'].value_counts())
 print(column_choice_test['Cabin_Letter'].value_counts())
 column_choice_training[['Cabin_Letter', 'Survived']].groupby(['Cabin_Letter'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
rf.fit(train[feats], train['nota_mat'] )
column_choice_training['Cabin_Number'] = column_choice_training['Cabin'].str.split(' ' ).str[-1].str[1:]
column_choice_test['Cabin_Number'] = column_choice_test['Cabin'].str.split(' ' ).str[-1].str[1:]
column_choice_training['Cabin_Number'] = column_choice_training['Cabin_Number'].replace(['0'], np.NaN)
 column_choice_test['Cabin_Number'] = column_choice_test['Cabin_Number'].replace(['0'], np.NaN)
 column_choice_training['Cabin_Number'] = column_choice_training['Cabin_Number'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 column_choice_test['Cabin_Number'] = column_choice_test['Cabin_Number'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 column_choice_training['Cabin_Number_cut'] = pd.qcut(column_choice_training['Cabin_Number'],4)
 column_choice_test['Cabin_Number_cut'] = pd.qcut(column_choice_test['Cabin_Number'],4)
 column_choice_training = pd.concat([column_choice_training.drop(['Cabin_Number', 'Cabin_Number_cut'], axis=1),
pd.get_dummies(column_choice_training['Cabin_Number_cut'], prefix = 'Cabin_num')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Cabin_Number', 'Cabin_Number_cut'], axis=1),
pd.get_dummies(column_choice_test['Cabin_Number_cut'], prefix = 'Cabin_num')], axis=1)
 column_choice_training.head()<categorify>
valid_preds = rf.predict(valid[feats])
 mean_squared_error(( valid['nota_mat']), valid_preds)**(1/2 )
column_choice_training['Cabin_Letter'] = column_choice_training['Cabin_Letter'].replace(['T'], 'N')
 column_choice_training = pd.concat([column_choice_training.drop(['Cabin', 'Cabin_Letter'], axis=1),
pd.get_dummies(column_choice_training['Cabin_Letter'], prefix = 'Cabin_Letter')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Cabin', 'Cabin_Letter'], axis=1),
pd.get_dummies(column_choice_test['Cabin_Letter'], prefix = 'Cabin_Letter')], axis=1)
 column_choice_training.head()<feature_engineering>
test['nota_mat'] = rf.predict(valid[feats] )
column_choice_training['Family'] = column_choice_training['SibSp'] + column_choice_training['Parch']
column_choice_test['Family'] = column_choice_test['SibSp'] + column_choice_test['Parch']
column_choice_training['Family'] = column_choice_training['Family'].replace([0], 'Alone')
 column_choice_training['Family'] = column_choice_training['Family'].replace([1, 2, 3], 'Little_Family')
 column_choice_training['Family'] = column_choice_training['Family'].replace([4, 5, 6, 7, 10], 'Big_Family')
 column_choice_test['Family'] = column_choice_test['Family'].replace([0], 'Alone')
 column_choice_test['Family'] = column_choice_test['Family'].replace([1, 2, 3], 'Little_Family')
 column_choice_test['Family'] = column_choice_test['Family'].replace([4, 5, 6, 7, 10], 'Big_Family')
 print(column_choice_training['Family'].value_counts())
 print(column_choice_test['Family'].value_counts())
 column_choice_training[['Family', 'Survived']].groupby(['Family'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 column_choice_training = pd.concat([column_choice_training.drop(['SibSp', 'Parch', 'Family'], axis=1),
pd.get_dummies(column_choice_training['Family'], prefix = 'Family')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['SibSp', 'Parch', 'Family'], axis=1),
pd.get_dummies(column_choice_test['Family'], prefix = 'Family')], axis=1 )<normalization>
test[['codigo_mun','nota_mat']].to_csv('rf_iesb_2019.csv', index=False )