Kaggle competition: Homework for Students
<SOS> metric: auc Kaggle data source: homework-for-students<feature_engineering>
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
all_data['Last_Name'] = all_data['Name'].apply(lambda x: str.split(x, ",")[0])
 all_data['Fare'].fillna(all_data['Fare'].mean() , inplace=True)
 DEFAULT_SURVIVAL_VALUE = 0.5
all_data['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in all_data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
all_data.loc[all_data['Family_Survival']!=0.5].shape[0] )<groupby>
pd.set_option('display.max_columns', 50 )
for _, grp_df in all_data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(all_data[all_data['Family_Survival']!=0.5].shape[0]))
train['Family_Survival'] = all_data['Family_Survival'][:891]
test['Family_Survival'] = all_data['Family_Survival'][891:]<groupby>
df_train = pd.read_csv('/kaggle/input/homework-for-students4plus/train.csv', index_col=0)
 df_test = pd.read_csv('/kaggle/input/homework-for-students4plus/test.csv', index_col=0 )
all_data['Ticket_Frequency'] = all_data.groupby('Ticket')['Ticket'].transform('count')
 train['Ticket_Frequency'] = all_data['Ticket_Frequency'][:891]
test['Ticket_Frequency'] = all_data['Ticket_Frequency'][891:]<categorify>
df_train_earliest = pd.merge(df_train['issue_d'].str[-4:], df_train,left_index=True, right_index=True)
 df_train_earliest = df_train_earliest.rename(columns = {'issue_d_x':'issue_d_year'})
 df_train_earliest = pd.merge(df_train['issue_d'].str[:3], df_train_earliest,left_index=True, right_index=True)
 df_train_earliest = df_train_earliest.rename(columns = {'issue_d':'issue_d_month'})
 df_train_earliest = df_train_earliest.rename(columns = {'issue_d_y':'issue_d'})
 df_test_earliest = pd.merge(df_test['issue_d'].str[-4:], df_test,left_index=True, right_index=True)
 df_test_earliest = df_test_earliest.rename(columns = {'issue_d_x':'issue_d_year'})
 df_test_earliest = pd.merge(df_test['issue_d'].str[:3], df_test_earliest,left_index=True, right_index=True)
 df_test_earliest = df_test_earliest.rename(columns = {'issue_d':'issue_d_month'})
 df_test_earliest = df_test_earliest.rename(columns = {'issue_d_y':'issue_d'})
 df_train = df_train_earliest
df_test = df_test_earliest
df_train['issue_d_month_mm'] = df_train['issue_d_month'].replace({'Jan':'01','Feb':'02','Mar':'03','Apr':'04','May':'05','Jun':'06','Jul':'07','Aug': '08','Sep': '09','Oct': '10','Nov': '11','Dec': '12'})
 df_test['issue_d_month_mm'] = df_test['issue_d_month'].replace({'Jan':'01','Feb':'02','Mar':'03','Apr':'04','May':'05','Jun':'06','Jul':'07','Aug': '08','Sep': '09','Oct': '10','Nov': '11','Dec': '12'})
 df_train['issue_d_yyyymm'] = df_train['issue_d_year'].str.cat(df_train['issue_d_month_mm'])
 df_test['issue_d_yyyymm'] = df_test['issue_d_year'].str.cat(df_test['issue_d_month_mm'])
 df_train['issue_d_yyyymm'] = df_train['issue_d_yyyymm'].fillna(-9999)
 df_train['issue_d_yyyymm'] = df_train['issue_d_yyyymm'].astype(int)
 df_test['issue_d_yyyymm'] = df_test['issue_d_yyyymm'].fillna(-9999)
 df_test['issue_d_yyyymm'] = df_test['issue_d_yyyymm'].astype(int)
 df_train['issue_d_year'] = df_train['issue_d_year'].fillna(-9999)
 df_train['issue_d_year'] = df_train['issue_d_year'].astype(int)
 df_test['issue_d_year'] = df_test['issue_d_year'].fillna(-9999)
 df_test['issue_d_year'] = df_test['issue_d_year'].astype(int)
 df_train['issue_d_month'] = df_train['issue_d_month'].replace({'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug': 8,'Sep': 9,'Oct': 10,'Nov': 11,'Dec': 12})
 df_test['issue_d_month'] = df_test['issue_d_month'].replace({'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug': 8,'Sep': 9,'Oct': 10,'Nov': 11,'Dec': 12})
 df_train['issue_d_month'] = df_train['issue_d_month'].fillna(-9999)
 df_train['issue_d_month'] = df_train['issue_d_month'].astype(int)
 df_test['issue_d_month'] = df_test['issue_d_month'].fillna(-9999)
 df_test['issue_d_month'] = df_test['issue_d_month'].astype(int)
all_data["Fare"] = all_data["Fare"].fillna(test["Fare"].median())
 all_data["Fare_per_one"]=all_data['Fare']/all_data['Ticket_Frequency']
all_data['FareBin'] = pd.qcut(all_data['Fare_per_one'], 16)
 label = LabelEncoder()
all_data['FareBin_Code'] = label.fit_transform(all_data['FareBin'])
 train['FareBin_Code'] = all_data['FareBin_Code'][:891]
test['FareBin_Code'] = all_data['FareBin_Code'][891:]
train.drop(['Fare'], 1, inplace=True)
 test.drop(['Fare'], 1, inplace=True )<categorify>
df_train = df_train[df_train['issue_d_yyyymm'] >= 201401]
all_data['AgeBin'] = pd.qcut(all_data['Age'], 6)
 label = LabelEncoder()
all_data['AgeBin_Code'] = label.fit_transform(all_data['AgeBin'])
 train['AgeBin_Code'] = all_data['AgeBin_Code'][:891]
test['AgeBin_Code'] = all_data['AgeBin_Code'][891:]
train.drop(['Age'], 1, inplace=True)
 test.drop(['Age'], 1, inplace=True )<drop_column>
cat = []
num = []
for col in df_train.columns:
if df_train[col].dtype == 'object':
cat.append(col)
 else:
num.append(col )
train['Sex'].replace(['male','female'],[0,1],inplace=True)
 test['Sex'].replace(['male','female'],[0,1],inplace=True)
 train.drop(['Name', 'PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin',
'Embarked'], axis = 1, inplace = True)
 test.drop(['Name','PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin',
'Embarked'], axis = 1, inplace = True )<prepare_x_and_y>
df_train['mths_since_last_delinq']+ df_train['mths_since_last_major_derog'] +df_train['mths_since_last_record']
y = train['Survived']
train_df = train.drop('Survived', 1)
 test_df = test.copy()<split>
df_train.loc[df_train['tot_cur_bal'].isnull() , 'tot_cur_bal'] = df_train[df_train['tot_cur_bal'].isnull() ]['loan_amnt']
df_test.loc[df_test['tot_cur_bal'].isnull() , 'tot_cur_bal'] = df_test[df_test['tot_cur_bal'].isnull() ]['loan_amnt']
my_train, my_test, my_y, my_res = train_test_split(train_df, y, test_size=0.1, random_state=42 )<train_model>
df_train['installment/annual_inc'] = df_train['installment']/(df_train['annual_inc']/12)
 df_test['installment/annual_inc'] = df_test['installment']/(df_test['annual_inc']/12)
model_try = RandomForestClassifier(n_estimators=300, max_depth=5, random_state=42)
 model_try.fit(my_train, my_y)
 preds = model_try.predict(my_test)
 print(accuracy_score(my_res, preds))<load_from_csv>
df_train['loan_amnt/installment'] = df_train['loan_amnt']/df_train['installment']
df_test['loan_amnt/installment'] = df_test['loan_amnt']/df_test['installment']
test = pd.read_csv('/kaggle/input/titanic/test.csv' )<save_to_csv>
sr_train_addr = df_train['addr_state'].str.cat(df_train['zip_code'])
 sr_test_addr = df_test['addr_state'].str.cat(df_test['zip_code'])
 df_train_addr = pd.DataFrame(sr_train_addr)
 df_test_addr = pd.DataFrame(sr_test_addr)
 df_train_addr = df_train_addr.rename(columns = {'addr_state':'address'})
 df_test_addr = df_test_addr.rename(columns = {'addr_state':'address'})
 df_train = pd.merge(df_train_addr, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_addr, df_test,left_index=True, right_index=True )
model = RandomForestClassifier(n_estimators=500, max_depth=5, random_state=42)
 model.fit(train_df, y)
 predictions = model.predict(test_df)
 output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print(predictions )<import_modules>
df_train['emp_title'] = df_train['emp_title'].str.lower()
df_test['emp_title'] = df_test['emp_title'].str.lower()
import numpy as np
import pandas as pd 
<set_options>
df_train_emp_title_split = df_train['emp_title'].str.split(' ', expand=True)
 df_test_emp_title_split = df_test['emp_title'].str.split(' ', expand=True)
 df_train_emp_title_split = df_train_emp_title_split.replace('',None)
 df_test_emp_title_split = df_test_emp_title_split.replace('',None)
 df_train_emp_title_split_count = df_train_emp_title_split.count(axis=1)
 df_test_emp_title_split_count = df_test_emp_title_split.count(axis=1)
 df_train_emp_title_split_count = pd.DataFrame(df_train_emp_title_split_count)
 df_test_emp_title_split_count = pd.DataFrame(df_test_emp_title_split_count)
 df_train_e = df_train_emp_title_split_count.rename(columns = {0:'emp_title_word_count'})
 df_test_e = df_test_emp_title_split_count.rename(columns = {0:'emp_title_word_count'})
 df_train = pd.merge(df_train_e, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_e, df_test,left_index=True, right_index=True)
sns.set(style='white')
 %matplotlib inline
SEED=101<concatenate>
df_train_nan_s = df_train.isnull().sum(axis=1)
 df_test_nan_s = df_test.isnull().sum(axis=1 )
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 def display_missing(df):
print(df.name)
 print('-'*77)
 for col in df.columns:
print('the number of missing values in {0}: {1}'.format(col, df[col].isnull().sum()))<load_from_csv>
df_train_nan = pd.DataFrame(df_train_nan_s)
 df_test_nan = pd.DataFrame(df_test_nan_s )
df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Testing Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
drop_list = []
passengerId = df_test['PassengerId']
print('Number of training example: {}'.format(df_train.shape[0]))
print('Number of testing example: {}
'.format(df_test.shape[0]))
print('Training X shape: {}'.format(df_train.shape))
print('Training Y shape: {}
'.format(df_train['Survived'].shape[0]))
print('Testing X shape: {}'.format(df_test.shape))
print('Testing Y shape: {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns)
<count_missing_values>
df_train_nan = df_train_nan.rename(columns = {0:'nan_count'})
 df_test_nan = df_test_nan.rename(columns = {0:'nan_count'} )
for df in dfs:
display_missing(df )<feature_engineering>
df_train = pd.merge(df_train_nan, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_nan, df_test,left_index=True, right_index=True )
age_median_by_pclass_and_sex = df_all.groupby(['Sex','Pclass'] ).median() ['Age']
for i in df_all['Pclass'].unique() :
for j in df_all['Sex'].unique() :
print('The median age of class{0} and {1} is: {2}'.format(i,j,age_median_by_pclass_and_sex[j][i]))
df_all['Age'] = df_all.groupby(['Sex','Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
display_missing(df_all )<feature_engineering>
def len_col_add(column,change_column,df_train,df_test):
df_train_change_len = df_train[column].str.len().fillna(0)
 df_test_change_len = df_test[column].str.len().fillna(0)
 df_train_change_len_next = pd.DataFrame(df_train_change_len)
 df_test_change_len_next = pd.DataFrame(df_test_change_len)
 df_train_change_len_next = df_train_change_len_next.rename(columns = {column:change_column})
 df_test_change_len_next = df_test_change_len_next.rename(columns = {column:change_column})
 df_train = pd.merge(df_train_change_len_next, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_change_len_next, df_test,left_index=True, right_index=True)
 return df_train,df_test
df_all['Embarked'] = df_all['Embarked'].fillna('S' )<filter>
df_train,df_test = len_col_add('emp_title','emp_title_len',df_train,df_test )
df_all.loc[df_all['Fare'].isnull() ]<categorify>
df_train,df_test = len_col_add('title','title_len',df_train,df_test )
med = df_all.groupby(['Pclass','Parch','SibSp'])['Fare'].median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med )<groupby>
df_train,df_test = len_col_add('home_ownership','home_ownership_len',df_train,df_test )
df_all['Deck'] = df_all['Cabin'].apply(lambda x: x[0] if pd.notnull(x)else 'M')
 df_all.groupby('Deck')['Survived'].mean()<drop_column>
df_train,df_test = len_col_add('purpose','purpose_len',df_train,df_test )
df_all_decks = df_all.groupby(['Deck','Pclass'] ).count().drop(columns=['Survived',
'Sex','SibSp','Age','Parch','Fare','Embarked', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
df_all_decks<groupby>
df_train_earliest = pd.merge(df_train['earliest_cr_line'].str[-4:], df_train,left_index=True, right_index=True)
 df_train_earliest = df_train_earliest.rename(columns = {'earliest_cr_line_x':'earliest_cr_line_year'})
 df_train_earliest = pd.merge(df_train['earliest_cr_line'].str[:3], df_train_earliest,left_index=True, right_index=True)
 df_train_earliest = df_train_earliest.rename(columns = {'earliest_cr_line':'earliest_cr_line_month'})
 df_train_earliest = df_train_earliest.rename(columns = {'earliest_cr_line_y':'earliest_cr_line'})
 df_test_earliest = pd.merge(df_test['earliest_cr_line'].str[-4:], df_test,left_index=True, right_index=True)
 df_test_earliest = df_test_earliest.rename(columns = {'earliest_cr_line_x':'earliest_cr_line_year'})
 df_test_earliest = pd.merge(df_test['earliest_cr_line'].str[:3], df_test_earliest,left_index=True, right_index=True)
 df_test_earliest = df_test_earliest.rename(columns = {'earliest_cr_line':'earliest_cr_line_month'})
 df_test_earliest = df_test_earliest.rename(columns = {'earliest_cr_line_y':'earliest_cr_line'})
 df_train = df_train_earliest
df_test = df_test_earliest
def get_pclass_dist(df):
deck_counts = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {}, 'G': {}, 'M': {}, 'T': {}}
decks = df.columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = df[deck][pclass][0]
deck_counts[deck][pclass] = count
except KeyError:
deck_counts[deck][pclass] = 0
df_decks = pd.DataFrame(deck_counts)
 deck_percentages = {}
for col in df_decks.columns:
deck_percentages[col] = [(count / df_decks[col].sum())* 100 for count in df_decks[col]]
return deck_counts, deck_percentages
all_deck_count, all_deck_per = get_pclass_dist(df_all_decks)
 print(all_deck_count)
 print(all_deck_per )<count_values>
df_train['earliest_cr_line_year'] = df_train['earliest_cr_line_year'].fillna(-9999)
 df_train['earliest_cr_line_year'] = df_train['earliest_cr_line_year'].astype(int)
 df_test['earliest_cr_line_year'] = df_test['earliest_cr_line_year'].fillna(-9999)
 df_test['earliest_cr_line_year'] = df_test['earliest_cr_line_year'].astype(int )
df_all['Deck'] = df_all['Deck'].replace('T','A')
 df_all['Deck'] = df_all['Deck'].replace(['A','B','C'],'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D','E'],'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F','G'],'FG')
 df_all['Deck'].value_counts()<drop_column>
df_train['earliest_cr_line_month'] = df_train['earliest_cr_line_month'].replace({'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug': 8,'Aug': 8,'Sep': 9,'Oct': 10,'Nov': 11,'Dec': 12})
 df_test['earliest_cr_line_month'] = df_test['earliest_cr_line_month'].replace({'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug': 8,'Aug': 8,'Sep': 9,'Oct': 10,'Nov': 11,'Dec': 12} )
display_missing(df_all)
 drop_list.append('Cabin')
 drop_list.append('PassengerId' )<split>
df_train['earliest_cr_line_month'] = df_train['earliest_cr_line_month'].fillna(-9999)
 df_train['earliest_cr_line_month'] = df_train['earliest_cr_line_month'].astype(int)
 df_test['earliest_cr_line_month'] = df_test['earliest_cr_line_month'].fillna(-9999)
 df_test['earliest_cr_line_month'] = df_test['earliest_cr_line_month'].astype(int )
df_all.drop(drop_list,axis=1,inplace=True)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train,df_test]<define_variables>
df_train['duration'] = df_train['issue_d_year'] - df_train['earliest_cr_line_year']
df_test['duration'] = df_test['issue_d_year'] - df_test['earliest_cr_line_year']
drop_list=[]<sort_values>
df_train.loc[df_train['duration'] > 100, 'duration'] = -9999
df_test.loc[df_test['duration'] > 100, 'duration'] = -9999
def corr_sort(df_train):
df_train_corr = df_train.corr().abs().unstack().sort_values(ascending=False ).reset_index()
df_train_corr.rename(columns={'level_0':'feature1','level_1':'feature2',0:'correlation'},inplace=True)
 df_train_corr = df_train_corr.iloc[::2]
return df_train_corr[(df_train_corr['correlation']>0.1)&(df_train_corr['correlation'] != 1)]<sort_values>
df_train_emp_len = df_train['emp_length'].fillna(-9999)
 df_test_emp_len = df_test['emp_length'].fillna(-9999)
 df_train_emp_len_next = pd.DataFrame(df_train_emp_len)
 df_test_emp_len_next = pd.DataFrame(df_test_emp_len)
 df_train_emp_len_next = df_train_emp_len_next.rename(columns = {'emp_length':'emp_length_num'})
 df_test_emp_len_next = df_test_emp_len_next.rename(columns = {'emp_length':'emp_length_num'})
 df_train_emp_len_next['emp_length_num'] = df_train_emp_len_next['emp_length_num'].replace({'< 1 year':0,'1 year':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5,'6 years':6,'7 years': 7,'8 years': 8,'9 years': 9,'10+ years': 10})
 df_test_emp_len_next['emp_length_num'] = df_test_emp_len_next['emp_length_num'].replace({'< 1 year':0,'1 year':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5,'6 years':6,'7 years': 7,'8 years': 8,'9 years': 9,'10+ years': 10})
 df_train_emp_len_next['emp_length_num'] = df_train_emp_len_next['emp_length_num'].astype(int)
 df_test_emp_len_next['emp_length_num'] = df_test_emp_len_next['emp_length_num'].astype(int)
 df_train = pd.merge(df_train_emp_len_next, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_emp_len_next, df_test,left_index=True, right_index=True)
corr_train = corr_sort(df_train)
 corr_test = corr_sort(df_test)
 print(corr_train)
 print(corr_test)
<count_missing_values>
df_train = pd.merge(df_train['zip_code'].str[:3], df_train,left_index=True, right_index=True)
 df_train['zip_code_x'] = df_train['zip_code_x'].astype(int)
 df_test = pd.merge(df_test['zip_code'].str[:3], df_test,left_index=True, right_index=True)
 df_test['zip_code_x'] = df_test['zip_code_x'].astype(int)
display_missing(df_all )<drop_column>
df_train = pd.merge(df_train['zip_code_y'].str[:1], df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test['zip_code_y'].str[:1], df_test,left_index=True, right_index=True)
drop_list.append('Fare')
 drop_list.append('Farec' )<drop_column>
gdp_df = pd.read_csv('.. /input/homework-for-students4plus/US_GDP_by_State.csv', index_col=0)
 desc_df = pd.read_csv('.. /input/homework-for-students4plus/description.csv', index_col=0)
 zip_df = pd.read_csv('.. /input/homework-for-students4plus/free-zipcode-database.csv', index_col=0)
 spi_df = pd.read_csv('.. /input/homework-for-students4plus/spi.csv', index_col=0)
 state_df = pd.read_csv('.. /input/homework-for-students4plus/statelatlong.csv', index_col=0)
drop_list.append('Age')
 drop_list.append('Agec' )<categorify>
X_test = df_test
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis=1)
 gc.collect()
family_map = {1: 'Alone', 2: 'Small', 3: 'Small', 4: 'Small', 5: 'Medium', 6: 'Medium', 7: 'Large', 8: 'Large', 11: 'Large'}
df_all['Family'] = df_all['FamilySize'].map(family_map)
 df_all['Family']<drop_column>
cat = []
num = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cat.append(col)
 else:
num.append(col)
drop_list.append('FamilySize')
 drop_list.append('Parch')
 drop_list.append('SibSp' )<categorify>
X_train['loan_amnt'] = X_train['loan_amnt'].apply(np.log1p)
 X_test['loan_amnt'] = X_test['loan_amnt'].apply(np.log1p)
def title_replace(row):
if row.Title == 'Mlle':
return 'Miss'
elif row.Title == 'Ms':
return 'Miss'
elif row.Title == 'Mme':
return 'Mrs'
elif row.Title in ['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona']:
return 'Rare'
else:
return row.Title 
<feature_engineering>
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 for col in cat:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
df_all['Title'] = df_all['Name'].str.split(',',expand=True)[1].str.split('.',expand=True)[0].str.strip()
df_all['Title'] = df_all['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 df_all['Title'] = df_all['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don', 'Rev'], 'Dr/Military/Noble/Clergy')
<feature_engineering>
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1<count_values>
import random
df_all['Title'].value_counts()<groupby>
clf_arry = []
skf_arry = []
for i in range(0,25):
rand_int =random.randint(1, 10000)
 clf_random = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.005, min_split_gain=0.0,
n_estimators=99999, n_jobs=-1, num_leaves=15, objective=None,
random_state= rand_int, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 skf_random = StratifiedKFold(n_splits=5, random_state= rand_int, shuffle=True)
 print(clf_random)
 print(skf_random)
 clf_arry.append(clf_random)
 skf_arry.append(skf_random)
df_all.groupby('Title')['Survived'].mean()<feature_engineering>
scores = []
y_pred_test = np.zeros(len(X_test))
for y in range(0,len(clf_arry)) :
clf = clf_arry[y]
skf = skf_arry[y]
print(clf)
 print(skf)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
print('⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️')
 print(y)
 print(i)
 print('⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️')
 X_train_, y_train_ = X_train.iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train.iloc[test_ix], y_train.iloc[test_ix]
clf.fit(X_train_, y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test)[:,1]
df_all['Surname'] = df_all['Name'].str.split(',',expand=True)[0].str.strip()
df_train, df_test = divide_df(df_all)
 common_surname = [each for each in df_test['Surname'].unique() if each in df_train['Surname'].unique() ]
df_train.groupby('Surname')['Survived'].median()
train_survived_dict_surname = df_train[df_train['FamilySize']>1].groupby('Surname')['Survived'].median().to_dict()
new_dict_surname = {}
for key in common_surname:
if key in train_survived_dict_surname.keys() :
new_dict_surname[key] = train_survived_dict_surname[key]
df_train['Sur_rate'] = df_train['Surname'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_test['Sur_rate'] = df_test['Surname'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_train['Sur_count'] = df_train['Surname'].apply(lambda x: 1 if x in new_dict_surname.keys() else 0)
 df_test['Sur_count'] = df_test['Surname'].apply(lambda x: 1 if x in new_dict_surname.keys() else 0 )<feature_engineering>
print(len(clf_arry))
y_pred = y_pred_test /(len(clf_arry)* 5)
 print('----------------------')
 print('各スコア：')
 x = 0
for i in scores:
x += i
print(i)
 print('----------------------')
 print(x)
 print('Ave.CV score：')
 print(x/(len(clf_arry)* 5))
common_ticket = [each for each in df_test['Ticket'].unique() if each in df_train['Ticket'].unique() ]
train_survived_dict_ticket = df_train[df_train['Ticket_Frequency']>1].groupby('Ticket')['Survived'].median().to_dict()
new_dict_ticket = {}
for key in common_ticket:
if key in train_survived_dict_ticket.keys() :
new_dict_ticket[key] = train_survived_dict_ticket[key]
df_train['tic_rate'] = df_train['Ticket'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_test['tic_rate'] = df_test['Ticket'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_train['tic_count'] = df_train['Ticket'].apply(lambda x: 1 if x in new_dict_ticket.keys() else 0)
 df_test['tic_count'] = df_test['Ticket'].apply(lambda x: 1 if x in new_dict_ticket.keys() else 0 )<feature_engineering>
imp = DataFrame(clf.booster_.feature_importance(importance_type='gain'), index = X_train.columns, columns=['importance'] ).sort_values(['importance'], ascending=False)
 imp
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['tic_rate'] + df['Sur_rate'])/ 2
df['Survival_Rate_NA'] =(df['tic_count'] + df['Sur_count'])/ 2<concatenate>
submission = pd.read_csv('/kaggle/input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )