Kaggle competition: Homework for Students
<SOS> metric: auc Kaggle data source: homework-for-students<feature_engineering>
plt.style.use('ggplot')
 %matplotlib inline
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv', index_col=0 )
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
df_train['issue_d_year']= pd.to_datetime(df_train['issue_d'] ).apply(lambda x: int(x.strftime('%Y')))
 df_test['issue_d_year']= pd.to_datetime(df_test['issue_d'] ).apply(lambda x: int(x.strftime('%Y')) )
df_val = df_val_1
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission<filter>
y_train = df_train.query('issue_d_year > 2014' ).loan_condition
X_train = df_train.query('issue_d_year > 2014' ).drop(['loan_condition'], axis=1)
 X_test = df_test
df_worldinfor[df_worldinfor['Country'] == 'Vietnam']<set_options>
X_train['nan_count'] = X_train.isnull().sum(axis=1)
 X_test['nan_count'] = X_test.isnull().sum(axis=1 )
sns.set() 
<load_from_csv>
for col in X_train.columns:
if X_train[col].isnull().any() :
X_train['nanflg_'+col] = X_train[col].isnull().astype(int)
 for col in X_test.columns:
if X_test[col].isnull().any() :
X_test['nanflg_'+col] = X_test[col].isnull().astype(int )
data_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 data_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 data_total = data_train.append(data_test, ignore_index=True)
 data_train_length = len(data_train.index)
 for c in ['Pclass', 'Sex', 'Embarked', 'SibSp', 'Parch', 'Survived']:
sns.catplot(x=c, kind='count', data=data_train)
 plt.show()
for c in ['Age', 'Fare']:
data_train[c].hist().set(xlabel=c)
 plt.show()
for c in ['Pclass', 'Sex', 'Embarked', 'SibSp', 'Parch']:
sns.catplot(x=c, y='Survived', kind='bar', data=data_train)
 plt.show()
for c in ['Age', 'Fare']:
g = sns.FacetGrid(data_train, col='Survived')
 g.map(plt.hist, c)
 plt.show()
corr = data_train.corr()
sns.boxplot(x='Pclass', y='Age', hue='Survived', data=data_train)
 plt.show()
sns.boxplot(x='Pclass', y='Fare', hue='Survived', data=data_train[data_train['Fare'] < 200])
 plt.show() 
<feature_engineering>
X_train['amnt_per_inc'] = X_train['loan_amnt']/X_train['annual_inc']
X_test['amnt_per_inc'] = X_test['loan_amnt']/X_test['annual_inc']
age_mean_by_pclass = data_total[['Pclass', 'Age']].groupby('Pclass' ).mean()
data_total['Age'] = data_total.apply(
 lambda row: age_mean_by_pclass.loc[row['Pclass'], 'Age'] if np.isnan(row['Age'])else row['Age'], axis=1)
 fare_median_by_pclass = data_total[['Pclass', 'Fare']].groupby('Pclass' ).median()
data_total['Fare'] = data_total.apply(
 lambda row: fare_median_by_pclass.loc[row['Pclass'], 'Fare'] if np.isnan(row['Fare'])else row['Fare'], axis=1)
 data_total['Embarked'].fillna(data_total['Embarked'].mode() [0], inplace=True)
 data_total.info()<feature_engineering>
X_train['amnt_per_revol'] = X_train['loan_amnt']/X_train['revol_bal']
X_test['amnt_per_revol'] = X_test['loan_amnt']/X_test['revol_bal']
data_total['isAlone'] =(( data_total['SibSp'] + data_total['Parch'])== 0)* 1
data_total['bigGroup'] =(( data_total['SibSp'] + data_total['Parch'])> 3)* 1
data_total['Title'] = data_total['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 data_total['Title'] = data_total['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')
 data_total['Title'] = data_total['Title'].replace(['Mlle', 'Ms', 'Mme'], 'Miss')
 age_bin_count = 10
data_total['Age_Bin'] = pd.cut(data_total['Age'], age_bin_count, labels=list(range(1,age_bin_count+1)))
 fare_bin_count = 5
data_total['Fare_Bin'] = pd.qcut(data_total['Fare'], fare_bin_count, labels=list(range(1,fare_bin_count+1)))
 enc = OneHotEncoder().fit(data_total[['Sex', 'Embarked', 'Title']])
 enc_features_arr = enc.transform(data_total[['Sex', 'Embarked', 'Title']] ).toarray()
enc_features_labels = [label for cat in enc.categories_ for label in cat]
enc_features_df = pd.DataFrame(enc_features_arr, columns=enc_features_labels ).drop(columns=['male', 'S', 'Other'])
 data_total = data_total.join(enc_features_df)
 data_total = data_total.drop(columns=['PassengerId', 'Name', 'Cabin', 'Ticket', 'SibSp', 'Parch', 'Age', 'Fare', 'Sex', 'Embarked', 'Title'])
 data_total.head()<train_model>
X_train['amnt_per_cur'] = X_train['loan_amnt']/X_train['tot_cur_bal']
X_test['amnt_per_cur'] = X_test['loan_amnt']/X_test['tot_cur_bal']
data_train = data_total[:data_train_length]
X = data_train.drop(columns=['Survived'], axis=1)
 y = data_train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 scaler = StandardScaler().fit(X_train)
 X_train = scaler.transform(X_train)
 X_test = scaler.transform(X_test )<define_search_space>
X_train['amnt_per_acc'] = X_train['loan_amnt']/X_train['total_acc']
X_test['amnt_per_acc'] = X_test['loan_amnt']/X_test['total_acc']
svc_params = [{'C': [0.2, 0.4, 0.5, 1, 2, 3, 4, 5], 'kernel': ['rbf'], 'gamma': [.10,.15,.20,.25,.30]}]
knn_params = [{'n_neighbors': [7, 8, 9, 10], 'weights': ['uniform'], 'algorithm': ['brute'], 'p': [2]}]
rfc_params = [{'n_estimators': [80, 90, 100, 110, 120, 130, 140], 'criterion': ['entropy']}]
classifier_names = ['SVC', 'KNN', 'RFC']
params = [svc_params, knn_params, rfc_params]
classifier_objs = [SVC() , KNeighborsClassifier() , RandomForestClassifier() ]
best_params = {}
for i in range(len(classifier_names)) :
classifier_objs[i].fit(X_train, y_train)
 grid_search = GridSearchCV(estimator=classifier_objs[i], param_grid=params[i], scoring='accuracy', cv=10 ).fit(X_train, y_train)
 best_params[classifier_names[i]] = grid_search.best_params_<choose_model_class>
X_train['revol_per_acc'] = X_train['revol_bal']/X_train['total_acc']
X_test['revol_per_acc'] = X_test['revol_bal']/X_test['total_acc']
classifier_tuned = [SVC(C=best_params['SVC']['C'], gamma=best_params['SVC']['gamma'], kernel=best_params['SVC']['kernel']),
KNeighborsClassifier(algorithm=best_params['KNN']['algorithm'], n_neighbors=best_params['KNN']['n_neighbors'],
p=best_params['KNN']['p'], weights=best_params['KNN']['weights']),
LogisticRegression() ,
RandomForestClassifier(criterion=best_params['RFC']['criterion'], n_estimators=best_params['RFC']['n_estimators']),
GaussianNB() ]
predictions_train = []
for clf in classifier_tuned:
clf.fit(X_train, y_train)
 y_pred = clf.predict(X_test)
 predictions_train.append(y_pred)
 print(clf)
 print(confusion_matrix(y_test, y_pred))
print('accuracy score: ' + str(accuracy_score(y_test, y_pred)))
 print('cross val score: ' + str(cross_val_score(clf, X_train, y_train ).mean()))
print('–––––––––')
 predictions_train = pd.DataFrame(predictions_train ).transpose()
predictions_train['y_pred_voted'] = predictions_train.mean(axis=1 ).round()
predictions_train_voted = predictions_train['y_pred_voted'].values
print('Result after voting:')
 print(confusion_matrix(y_test, predictions_train_voted))
print(accuracy_score(y_test, predictions_train_voted))<import_modules>
X_train['revol_per_util'] = X_train['revol_bal']/X_train['revol_util']
X_test['revol_per_util'] = X_test['revol_bal']/X_test['revol_util']
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from itertools import cycle, islice
import seaborn as sb
import matplotlib.dates as dates
import datetime as dt
from sklearn import preprocessing
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
import plotly.graph_objects as go
import plotly_express as px
from sklearn.preprocessing import OrdinalEncoder<load_from_csv>
X_train['revol_by_acc'] = X_train['revol_bal']*X_train['total_acc']
X_test['revol_by_acc'] = X_test['revol_bal']*X_test['total_acc']
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv")
 train.head()<data_type_conversions>
X_train['amnt_by_inq'] = X_train['loan_amnt']*X_train['inq_last_6mths']
X_test['amnt_by_inq'] = X_test['loan_amnt']*X_test['inq_last_6mths']
train['Date'] = pd.to_datetime(train['Date'], format = '%Y-%m-%d')
 test['Date'] = pd.to_datetime(test['Date'], format = '%Y-%m-%d' )<feature_engineering>
X_train['nan_cnt'] = X_train.isnull().sum(axis=1)
 X_test['nan_cnt'] = X_test.isnull().sum(axis=1 )
def create_features(df):
df['day'] = df['Date'].dt.day
df['month'] = df['Date'].dt.month
df['dayofweek'] = df['Date'].dt.dayofweek
df['dayofyear'] = df['Date'].dt.dayofyear
df['quarter'] = df['Date'].dt.quarter
df['weekofyear'] = df['Date'].dt.weekofyear
return df<categorify>
X_train['issue_d'] = pd.to_datetime(X_train.issue_d, format="%b-%Y")
 X_test['issue_d'] = pd.to_datetime(X_test.issue_d, format="%b-%Y" )
def categoricalToInteger(df):
df.Province_State.fillna('NaN', inplace=True)
 oe = OrdinalEncoder()
df[['Province_State','Country_Region']] = oe.fit_transform(df.loc[:,['Province_State','Country_Region']])
 return df<drop_column>
X_train['issue_d_month'] = X_train['issue_d'].dt.month.astype(str)
 X_test['issue_d_month'] = X_test['issue_d'].dt.month.astype(str )
columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear','Province_State', 'Country_Region','ConfirmedCases','Fatalities']
df_train = df_train[columns]<feature_engineering>
X_train['issue_d'] = X_train['issue_d'].astype(int)
 X_test['issue_d'] = X_test['issue_d'].astype(int )
df_test = categoricalToInteger(test)
 df_test = create_features(test)
 columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear','Province_State', 'Country_Region']<prepare_x_and_y>
X_train['earliest_cr_line'] = pd.to_datetime(X_train.earliest_cr_line, format="%b-%Y")
 X_test['earliest_cr_line'] = pd.to_datetime(X_test.earliest_cr_line, format="%b-%Y" )
submission = []
for country in df_train.Country_Region.unique() :
df_train1 = df_train[df_train["Country_Region"]==country]
for state in df_train1.Province_State.unique() :
df_train2 = df_train1[df_train1["Province_State"]==state]
train = df_train2.values
X_train, y_train = train[:,:-2], train[:,-2:]
model1 = XGBRegressor(n_estimators=1100)
 model1.fit(X_train, y_train[:,0])
 model2 = XGBRegressor(n_estimators=1100)
 model2.fit(X_train, y_train[:,1])
 df_test1 = df_test[(df_test["Country_Region"]==country)&(df_test["Province_State"] == state)]
ForecastId = df_test1.ForecastId.values
df_test2 = df_test1[columns]
y_pred1 = np.round(model1.predict(df_test2.values),5)
 y_pred2 = np.round(model2.predict(df_test2.values),5)
 for i in range(len(y_pred1)) :
d = {'ForecastId':ForecastId[i], 'ConfirmedCases':y_pred1[i], 'Fatalities':y_pred2[i]}
submission.append(d )<save_to_csv>
X_train['earliest_cr_line_year'] = X_train['earliest_cr_line'].dt.year
X_test['earliest_cr_line_year'] = X_test['earliest_cr_line'].dt.year
df_submit = pd.DataFrame(submission)
 df_submit.to_csv(r'submission.csv', index=False )<load_from_csv>
X_train['earliest_cr_line_month'] = X_train['earliest_cr_line'].dt.month.astype(str)
 X_test['earliest_cr_line_month'] = X_test['earliest_cr_line'].dt.month.astype(str )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<define_variables>
X_train['earliest_cr_line'] = X_train['earliest_cr_line'].astype(int)
 X_test['earliest_cr_line'] = X_test['earliest_cr_line'].astype(int )
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )<define_variables>
X_train['relative_earliest_cr_line'] = X_train['issue_d'] - X_train['earliest_cr_line']
X_test['relative_earliest_cr_line'] = X_test['issue_d'] - X_test['earliest_cr_line']
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )<groupby>
lst = [X_train]
X_train['emp_length_int'] = np.nan
for col in lst:
col.loc[col['emp_length'] == '10+ years', "emp_length_int"] = 10
col.loc[col['emp_length'] == '9 years', "emp_length_int"] = 9
col.loc[col['emp_length'] == '8 years', "emp_length_int"] = 8
col.loc[col['emp_length'] == '7 years', "emp_length_int"] = 7
col.loc[col['emp_length'] == '6 years', "emp_length_int"] = 6
col.loc[col['emp_length'] == '5 years', "emp_length_int"] = 5
col.loc[col['emp_length'] == '4 years', "emp_length_int"] = 4
col.loc[col['emp_length'] == '3 years', "emp_length_int"] = 3
col.loc[col['emp_length'] == '2 years', "emp_length_int"] = 2
col.loc[col['emp_length'] == '1 year', "emp_length_int"] = 1
col.loc[col['emp_length'] == '< 1 year', "emp_length_int"] = 0.5
col.loc[col['emp_length'] == 'n/a', "emp_length_int"] = 0
lst_test = [X_test]
X_test['emp_length_int'] = np.nan
for col in lst_test:
col.loc[col['emp_length'] == '10+ years', "emp_length_int"] = 10
col.loc[col['emp_length'] == '9 years', "emp_length_int"] = 9
col.loc[col['emp_length'] == '8 years', "emp_length_int"] = 8
col.loc[col['emp_length'] == '7 years', "emp_length_int"] = 7
col.loc[col['emp_length'] == '6 years', "emp_length_int"] = 6
col.loc[col['emp_length'] == '5 years', "emp_length_int"] = 5
col.loc[col['emp_length'] == '4 years', "emp_length_int"] = 4
col.loc[col['emp_length'] == '3 years', "emp_length_int"] = 3
col.loc[col['emp_length'] == '2 years', "emp_length_int"] = 2
col.loc[col['emp_length'] == '1 year', "emp_length_int"] = 1
col.loc[col['emp_length'] == '< 1 year', "emp_length_int"] = 0.5
col.loc[col['emp_length'] == 'n/a', "emp_length_int"] = 0
train_data['age_group'] = pd.cut(train_data.Age, bins=[0, 10, 20, 30, 40, 50, 60, 70, 80])
 Gender_grp = train_data.groupby(['Sex'])
 percent_present = dict(Gender_grp.get_group('female' ).age_group.value_counts(normalize=True))
num_present = dict(Gender_grp.get_group('female' ).age_group.value_counts())
 survivors = Gender_grp.get_group('female')[Gender_grp.get_group('female' ).Survived == 1]
age_survived = {}
for i in num_present:
age_survived[i] = survivors.loc[survivors.age_group==i]["Survived"].count()
list_of_dict = [age_survived,num_present]
female_survivors = pd.DataFrame(list_of_dict, index = ["Survived","Total"] ).transpose()
female_survivors["Normalized"] = female_survivors['Survived']/female_survivors['Total']
female_survivors<feature_engineering>
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )
num_present = dict(Gender_grp.get_group('male' ).age_group.value_counts())
 print(num_present)
 survivors_1 = Gender_grp.get_group('male')[Gender_grp.get_group('male' ).Survived == 1]
age_survived_1 = {}
for i in num_present:
age_survived[i] = survivors_1.loc[survivors_1.age_group==i]["Survived"].count()
list_of_dict = [age_survived,num_present]
male_survivors = pd.DataFrame(list_of_dict, index = ["Survived","Total"] ).transpose()
male_survivors["Normalized"] = male_survivors['Survived']/male_survivors['Total']
male_survivors<groupby>
col = 'purpose'
encoder = OneHotEncoder()
enc_train = encoder.fit_transform(pd.DataFrame(X_train[col].values))
enc_test = encoder.transform(pd.DataFrame(X_test[col].values))
print("Number of missing values for females",Gender_grp.get_group('female' ).Age.isnull().sum())
 print("Total for females",Gender_grp.get_group('female' ).PassengerId.count())
 print("Number of missing values for males",Gender_grp.get_group('male' ).Age.isnull().sum())
 print("Total for males", Gender_grp.get_group('male' ).PassengerId.count() )<feature_engineering>
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )
<groupby>
oelist = cats
for c in oelist:
oe=OrdinalEncoder()
oe.fit(pd.DataFrame(X_train[c]))
X_train[c]=oe.fit_transform(pd.DataFrame(X_train[c]))
X_test[c]=oe.transform(pd.DataFrame(X_test[c]))
pclass_group = train_data.groupby('Pclass')
<create_dataframe>
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby(['emp_title'])['loan_condition'].mean()
X_test['emp_title'] = X_test['emp_title'].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby(['emp_title'])['loan_condition'].mean()
enc_train.iloc[val_ix] = X_val['emp_title'].map(summary)
 X_train['emp_title'] = enc_train
num_present_pclass_1 = dict(pclass_group.get_group(1 ).age_group.value_counts())
 survived_age_pclass_1 = {}
for i in num_present_pclass_1:
value = pclass_group.get_group(1 ).loc[pclass_group.get_group(1 ).age_group == i]['Survived']
survived_age_pclass_1[i]=sum(value)
 list_of_pclass_1=[survived_age_pclass_1,num_present_pclass_1]
pclass_1 = pd.DataFrame(list_of_pclass_1,index=['Survived','Total'] ).transpose()
pclass_1['normalised'] = pclass_1['Survived']/pclass_1['Total']<create_dataframe>
X_train.fillna(X_train.mean() , axis=0, inplace=True)
 X_test.fillna(X_train.mean() , axis=0, inplace=True )
num_present_pclass_2 = dict(pclass_group.get_group(2 ).age_group.value_counts())
 survived_age_pclass_2 = {}
for i in num_present_pclass_2:
value = pclass_group.get_group(2 ).loc[pclass_group.get_group(2 ).age_group == i]['Survived']
survived_age_pclass_2[i]=sum(value)
 list_of_pclass_2=[survived_age_pclass_2,num_present_pclass_2]
pclass_2 = pd.DataFrame(list_of_pclass_2,index=['Survived','Total'] ).transpose()
pclass_2['normalised'] = pclass_2['Survived']/pclass_2['Total']<create_dataframe>
X_temp = X_train
y_target = y_train
X_train_bin = X_temp
X_test_bin = X_test
num_present_pclass_3 = dict(pclass_group.get_group(3 ).age_group.value_counts())
 survived_age_pclass_3 = {}
for i in num_present_pclass_3:
value = pclass_group.get_group(3 ).loc[pclass_group.get_group(3 ).age_group == i]['Survived']
survived_age_pclass_3[i]=sum(value)
 list_of_pclass_3=[survived_age_pclass_3,num_present_pclass_3]
pclass_3 = pd.DataFrame(list_of_pclass_3,index=['Survived','Total'] ).transpose()
pclass_3['normalised'] = pclass_3['Survived']/pclass_3['Total']<groupby>
X_train_, X_val, y_train_, y_val= train_test_split(X_train_bin, y_target, test_size=0.05, random_state=71 )
gender_class_group = train_data.groupby(['Sex','Pclass'] )<create_dataframe>
clf = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=9999, n_jobs=-1, num_leaves=15, objective=None,
random_state=71, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0 )
num_present_gender_class_1 = dict(gender_class_group.get_group(( 'female',1)).age_group.value_counts())
 survived_1 = {}
for i in num_present_gender_class_1:
value = gender_class_group.get_group(( 'female',1)).loc[gender_class_group.get_group(( 'female',1)).age_group==i]['Survived']
survived_1[i] = sum(value)
 list_1 = [survived_1,num_present_gender_class_1]
gender_class_1 = pd.DataFrame(list_1, index = ['Survived','Total'] ).transpose()
gender_class_1<groupby>
%%time
clf.fit(X_train_, y_train_, early_stopping_rounds=20, eval_metric='auc', eval_set=[(X_val, y_val)] )
print("Number of missing values for females",gender_class_group.get_group(( 'female',1)).Age.isnull().sum())
 print("Total for females",gender_class_group.get_group(( 'female',1)).PassengerId.count() )<feature_engineering>
imp = DataFrame(clf.booster_.feature_importance(importance_type='gain'), index = X_train_bin.columns, columns=['importance'] ).sort_values(['importance'], ascending=False)
 imp
<groupby>
use_col = imp.index[:55]
print("Number of missing values for females",gender_class_group.get_group(( 'female',1)).Age.isnull().sum())
 print("Total for females",gender_class_group.get_group(( 'female',1)).PassengerId.count() )<create_dataframe>
%%time
clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=20, eval_metric='auc', eval_set=[(X_val[use_col], y_val)] )
num_present_gender_class_2 = dict(gender_class_group.get_group(( 'female',2)).age_group.value_counts())
 survived_2 = {}
for i in num_present_gender_class_2:
value = gender_class_group.get_group(( 'female',2)).loc[gender_class_group.get_group(( 'female',2)).age_group==i]['Survived']
survived_2[i] = sum(value)
 list_2 = [survived_2,num_present_gender_class_2]
gender_class_2 = pd.DataFrame(list_2, index = ['Survived','Total'] ).transpose()
gender_class_2<groupby>
scores = []
y_pred_test = np.zeros(len(X_test[use_col]))
skf = StratifiedKFold(n_splits=10, random_state=2020, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train[use_col], y_train))):
X_train_, y_train_ = X_train[use_col].iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train[use_col].iloc[test_ix], y_train.iloc[test_ix ]
clf = LGBMClassifier(max_depth=50, n_estimators=500, num_leaves=100, learning_rate=0.1, silent=False)
 clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores = np.append(scores, roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test[use_col])[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean() )
print("Number of missing values for females",gender_class_group.get_group(( 'female',2)).Age.isnull().sum())
 print("Total for females",gender_class_group.get_group(( 'female',2)).PassengerId.count() )<create_dataframe>
skf = StratifiedKFold(n_splits=10, random_state=71, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train[use_col], y_train))):
X_train_, y_train_ = X_train[use_col].iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train[use_col].iloc[test_ix], y_train.iloc[test_ix ]
clf = LGBMClassifier(max_depth=50, n_estimators=500, num_leaves=100, learning_rate=0.1, silent=False)
 clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores = np.append(scores, roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test[use_col])[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean() )
num_present_gender_class_3 = dict(gender_class_group.get_group(( 'female',3)).age_group.value_counts())
 survived_3 = {}
for i in num_present_gender_class_3:
value = gender_class_group.get_group(( 'female',3)).loc[gender_class_group.get_group(( 'female',3)).age_group==i]['Survived']
survived_3[i] = sum(value)
 list_3 = [survived_3,num_present_gender_class_3]
gender_class_3 = pd.DataFrame(list_3, index = ['Survived','Total'] ).transpose()
gender_class_3<groupby>
skf = StratifiedKFold(n_splits=10, random_state=42, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train[use_col], y_train))):
X_train_, y_train_ = X_train[use_col].iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train[use_col].iloc[test_ix], y_train.iloc[test_ix ]
clf = LGBMClassifier(max_depth=50, n_estimators=500, num_leaves=100, learning_rate=0.1, silent=False)
 clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores = np.append(scores, roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test[use_col])[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean() )
print("Number of missing values for females",gender_class_group.get_group(( 'female',3)).Age.isnull().sum())
 print("Total for females",gender_class_group.get_group(( 'female',3)).PassengerId.count() )<create_dataframe>
skf = StratifiedKFold(n_splits=10, random_state=1, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train[use_col], y_train))):
X_train_, y_train_ = X_train[use_col].iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train[use_col].iloc[test_ix], y_train.iloc[test_ix ]
clf = LGBMClassifier(max_depth=50, n_estimators=500, num_leaves=100, learning_rate=0.1, silent=False)
 clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores = np.append(scores, roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test[use_col])[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean() )
num_present_gender_class_4 = dict(gender_class_group.get_group(( 'male',1)).age_group.value_counts())
 survived_4 = {}
for i in num_present_gender_class_4:
value = gender_class_group.get_group(( 'male',1)).loc[gender_class_group.get_group(( 'male',1)).age_group==i]['Survived']
survived_4[i] = sum(value)
 list_4 = [survived_4,num_present_gender_class_4]
gender_class_4 = pd.DataFrame(list_4, index = ['Survived','Total'] ).transpose()
gender_class_4<groupby>
skf = StratifiedKFold(n_splits=10, random_state=0, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train[use_col], y_train))):
X_train_, y_train_ = X_train[use_col].iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train[use_col].iloc[test_ix], y_train.iloc[test_ix ]
clf = LGBMClassifier(max_depth=50, n_estimators=500, num_leaves=100, learning_rate=0.1, silent=False)
 clf.fit(X_train_[use_col], y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores = np.append(scores, roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test[use_col])[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean() )
print("Number of missing values for males",gender_class_group.get_group(( 'male',1)).Age.isnull().sum())
 print("Total for males",gender_class_group.get_group(( 'male',1)).PassengerId.count() )<create_dataframe>
y_pred_test /= 50
<groupby><EOS> 
submission = pd.read_csv('.. /input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred_test
submission.to_csv('submission.csv' )