Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
pd.set_option('display.width', 500)
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.notebook_repr_html', True)
 sns.set(style="whitegrid")
 warnings.filterwarnings('ignore')
 InteractiveShell.ast_node_interactivity = "all"
opt = OptimizedRounder()
opt.fit(lgb_xgb, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_test.mean(1), coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_xgb_lgb.csv", index=False)
 submission.head()<save_to_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 Test = pd.read_csv(".. /input/titanic/test.csv")
 combine = [train, Test]
combined = pd.concat(combine )
opt = OptimizedRounder()
opt.fit(lgb_xgb_nn, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb_nn, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb_nn, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_nn_test, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission.csv", index=False)
 submission.head()<save_to_csv>
train.count()
opt = OptimizedRounder()
opt.fit(nn_preds * 4, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(nn_preds * 4, coeff)
 qwk = quadratic_weighted_kappa(target, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(nn_preds * 4, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(nn_preds_test * 4, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_nn.csv", index=False)
 submission.head()<import_modules>
print('Train columns with null values:',train.isnull().sum() , sep = '
')
 print("-"*42)
 print('Test/Validation columns with null values:', Test.isnull().sum() ,sep = '
')
 print("-"*42 )
print("Python version: {}".format(sys.version))
print("pandas version: {}".format(pd.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("NumPy version: {}".format(np.__version__))
print("SciPy version: {}".format(sp.__version__))
print("IPython version: {}".format(IPython.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
warnings.filterwarnings('ignore')
 warnings.simplefilter(action="ignore", category=ConvergenceWarning)
 print('-'*25)
 print(check_output(["ls", ".. /input"] ).decode("utf8")) 
<import_modules>
!pip install -U -q pandas-profiling
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('darkgrid')
 pylab.rcParams['figure.figsize'] = 12,8<choose_model_class>
def print_percentage(df,col_name,col_values):
for x in col_values:
group = df.loc[df[col_name]==x]
print('{0} survival rate: {1:.3f}'.format(x,(group['Survived'].sum() /group.shape[0])) )
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
def train_model(data, MLA_list = MLA, print_feature_table=False, print_feature_score=False, top_n=8):
target = data['AdoptionSpeed']
X_train = data.drop(['AdoptionSpeed'],axis=1)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA cohen_kappa_score','MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data['AdoptionSpeed']
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 row_index = 0
for alg in MLA_list:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 cv_results = model_selection.cross_validate(alg, X_train, target, cv = kf, scoring=kappa_score)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA cohen_kappa_score'] = cv_results['test_score'].mean()
row_index+=1
MLA_compare.sort_values(by = ['MLA cohen_kappa_score'], ascending = False, inplace = True)
 sns.barplot(x='MLA cohen_kappa_score', y = 'MLA Name', data = MLA_compare, color = 'b')
 plt.title('Machine Learning Algorithm Accuracy Score
')
 plt.xlabel('Accuracy Score(%)')
 plt.ylabel('Algorithm')
 if print_feature_table:
for alg in MLA_list:
alg.fit(X_train, target)
 if hasattr(alg, 'feature_importances_'):
feat_imp = pd.DataFrame({'importance':alg.feature_importances_})
 feat_imp['feature'] = X_train.columns
feat_imp.sort_values(by='importance', ascending=False, inplace=True)
 feat_imp = feat_imp.iloc[:top_n]
feat_imp.sort_values(by='importance', inplace=True)
 feat_imp = feat_imp.set_index('feature', drop=True)
 feat_imp.plot.barh(title=alg.__class__.__name__)
 plt.xlabel('Feature Importance Score')
 plt.show()
if print_feature_score:
print("Top {} features in descending order of importance".format(top_n))
display(feat_imp.sort_values(by='importance', ascending=False))
return MLA_compare 
<load_from_csv>
train.Parch.value_counts()
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 train['dataset_type'] = 'train'
test['dataset_type'] = 'test'<drop_column>
print_percentage(train,'Parch',[0,1,2,3,4,5,6] )
useColumns = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed'] 
<feature_engineering>
train.SibSp.value_counts()
train['Name'] = train['Name'].fillna('Unnamed')
 test['Name'] = test['Name'].fillna('Unnamed')
 train['has_name'] = train['Name'].apply(lambda x: 0 if x == 'Unnamed' else 1)
 test['has_name'] = test['Name'].apply(lambda x: 0 if x == 'Unnamed' else 1 )<feature_engineering>
print_percentage(train,'SibSp',[0,1,2,3,4,5,8] )
train['has_name'] = train['Name'].apply(lambda x: 0 if x == 'No Name' or x == 'Unnamed' else 1)
 test['has_name'] = test['Name'].apply(lambda x: 0 if x == 'No Name' or x == 'Unnamed' else 1 )<choose_model_class>
print('Amount of missing data in Fare for training set:', train.Fare.isnull().sum())
 print('Amount of missing data in Fare for test set:', Test.Fare.isnull().sum() )
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name'] 
<count_values>
Test[Test['Fare'].isnull() ]
def top_names(df, top_percent):
df_withnames = df[df.has_name != 0]
items = df_withnames.shape[0]
top_names = []
counter = 0
for i,v in df_withnames.Name.value_counts().items() :
if(counter/items)>top_percent:
break
top_names.append(i)
 counter = counter + v
return top_names<feature_engineering>
median_fare = Test.groupby(['Pclass', 'Parch'] ).Fare.median() [3][0]
Test['Fare'] = Test['Fare'].fillna(median_fare )
topnames = top_names(train.append(test), 0.2)
 train['has_topname'] = train['Name'].apply(lambda row: 1 if row in topnames else 0)
 test['has_topname'] = test['Name'].apply(lambda row: 1 if row in topnames else 0 )<categorify>
print('Amount of missing data in Embarked for train:', train.Embarked.isnull().sum())
 print('Amount of missing data in Embarked for test:', Test.Embarked.isnull().sum() )
cut_points = [1,2,3,4,8,12,24,36,48]
labels = ["1m","2m","3m","4m","8m","1y","2y","3y","4y","5y and over"]
train["age_bin"] = binning(train["Age"], cut_points, labels)
 labels = [1,2,3,4,5,6,7,8,9,10]
train["age_bin"] = binning(train["Age"], cut_points, labels)
 test["age_bin"] = binning(test["Age"], cut_points, labels)
 label = LabelEncoder()
train['age_bin'] = label.fit_transform(train['age_bin'])
 test['age_bin'] = label.fit_transform(test['age_bin'] )<choose_model_class>
train[train['Embarked'].isnull() ]
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin'] 
<create_dataframe>
train['Embarked'] = train['Embarked'].fillna('S' )
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='kmeans')
 train['age_bin_kmeans'] = pd.DataFrame(binner.fit_transform(train[['Age']].copy()))
test['age_bin_kmeans'] = pd.DataFrame(binner.fit_transform(test[['Age']].copy()))<create_dataframe>
print('Amount of missing data in Age for train:', train.Age.isnull().sum())
 print('Amount of missing data in Age for test:', Test.Age.isnull().sum() )
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='quantile')
 train['age_bin_quantile'] = pd.DataFrame(binner.fit_transform(train[['Age']].copy()))
test['age_bin_quantile'] = pd.DataFrame(binner.fit_transform(test[['Age']].copy()))<train_model>
copy5 = train.copy()
missing_age_rows2 = copy5.Age.isna()
missing_age_rows2
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin', 'age_bin_kmeans', 'age_bin_quantile'] 
<define_variables>
age_by_pclass_SibSp = copy5.groupby(['Pclass', 'SibSp'] ).median() ['Age']
age_by_pclass_SibSp
cat_human_age = {1:0.5, 2:3, 3:4, 4:6, 5:8, 6:10, 7:12, 8:14, 9:15, 10:16, 11:17, 12:18, 24:24, 48:35, 72:42, 96:50, 120:60, 144:70, 168:80, 192:84 }
small_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:15, 24:23, 48:32, 72:40, 96:48, 120:56, 144:64, 168:72, 192:80 }
normal_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:15, 24:24, 48:34, 72:42, 96:51, 120:60, 144:69, 168:78, 192:87 }
big_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:14, 24:22, 48:34, 72:45, 96:55, 120:66, 144:77, 168:88, 192:99 }
def human_age(row):
months = row['Age']
if months == 0:
return 0
if row['Type'] == 2:
if cat_human_age.get(months)is not None:
return cat_human_age.get(months)
 else:
if months < 25:
return 25
else:
return(25 +(( months/12)- 2)* 4)
 elif row['Type'] == 1 and row['MaturitySize'] == 1:
if small_dog_human_age.get(months)is not None:
return small_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 elif row['Type'] == 1 and row['MaturitySize'] == 3:
if big_dog_human_age.get(months)is not None:
return big_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 if normal_dog_human_age.get(months)is not None:
return normal_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 def lifestage(row):
age = row['human_age']
if age < 10:
return 'Kitten/Puppy'
elif age < 25:
return 'Junior'
elif age < 40:
return 'Prime'
elif age < 60:
return 'Mature'
elif age < 74:
return 'Senior'
return 'Geriatic'<feature_engineering>
age_by_pclass_SibSp[1].index.tolist()
age_by_pclass_SibSp[3][8] = age_by_pclass_SibSp[3][5]
age_by_pclass_SibSp
train['human_age'] = train.apply(human_age, axis=1)
 train['lifestage'] = train.apply(lifestage, axis=1)
 test['human_age'] = test.apply(human_age, axis=1)
 test['lifestage'] = test.apply(lifestage, axis=1 )<choose_model_class>
copy5['Age'] = copy5.groupby(['Pclass', 'SibSp'])[
'Age'].apply(lambda x: x.fillna(x.median()))
copy5['Age'] = copy5.Age.fillna(11)
 copy5.info()
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age'] 
<categorify>
copy2 = train.copy()
copy2['Age'].fillna(copy2['Age'].median() ,inplace = True )
mapper = {'Kitten/Puppy':1, 'Junior':2, 'Prime':3, 'Mature':4,'Senior':5,'Geriatic':6}
train.lifestage.replace(mapper, inplace=True)
 test.lifestage.replace(mapper, inplace=True )<define_variables>
train = copy5
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage'] 
<feature_engineering>
test_age_by_pclass_SibSp = Test.groupby(['Pclass', 'SibSp'] ).median() ['Age']
test_age_by_pclass_SibSp
Test['Age'] = Test.groupby(['Pclass', 'SibSp'])['Age'].apply(lambda x: x.fillna(x.median()))
Test.info()
def mixed_breed(row):
if row['Breed1'] == 307:
return 1
elif row['Breed2'] == 0:
return 0
elif row['Breed2'] != row['Breed1']:
return 1
else:
return 0
train['mixed_breed'] = train.apply(mixed_breed, axis=1)
 test['mixed_breed'] = test.apply(mixed_breed, axis=1 )<categorify>
def detect_outliers(df, n, features):
outliers_indices = []
for col in features:
Q1 = np.nanpercentile(df[col], 25)
 Q3 = np.nanpercentile(df[col], 75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)
 |(df[col] > Q3 + outlier_step)].index
print(col,Q1-outlier_step,Q3+outlier_step)
 outliers_indices.extend(outlier_list_col)
 outliers_indices = Counter(outliers_indices)
 multiple_outliers = list(k for k, v in outliers_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train, 2, ["Age", "SibSp", "Parch", "Fare"])
top_1 = [
ensemble.GradientBoostingClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed'] 
<groupby>
Outliers_to_drop
train.loc[Outliers_to_drop]
train.groupby('Breed1')["AdoptionSpeed"].value_counts().head(5 )<groupby>
def outlier_treatment(datacolumn):
sorted(datacolumn)
 Q1, Q3 = np.nanpercentile(datacolumn,[25,75])
 IQR = Q3-Q1
lower_range = Q1-(1.5*IQR)
 upper_range = Q3+(1.5*IQR)
 return lower_range, upper_range
print('Upper and Lower bound for Age:',outlier_treatment(train.Age), sep = '
')
 print('Upper and Lower bound for Fare:',outlier_treatment(train.Fare), sep = '
')
 print('Upper and Lower bound for Parch:',outlier_treatment(train.Parch), sep = '
')
 print('Upper and Lower bound for Siblings:',outlier_treatment(train.SibSp), sep = '
')
subtrain = train[train.Breed1 == 0]
cumulative_sum = subtrain.groupby('Breed1')["AdoptionSpeed"].cumsum() - subtrain["AdoptionSpeed"]
cumulative_count = subtrain.groupby('Breed1' ).cumcount()
print(cumulative_sum)
 print(cumulative_count)
 cumulative_sum/cumulative_count<categorify>
print('Number of Unique values for Name is', len(train.Name.unique()))
print('Number of Unique values for PassengerID is', len(train.PassengerId.unique()))
print('Number of Unique values for Fare is', len(train.Fare.unique()))
print('Number of Unique values for Survived is', len(train.Survived.unique()))
print('Number of Unique values for Pclass is', len(train.Pclass.unique()))
print('Number of Unique values for Parch is', len(train.Parch.unique()))
print('Number of Unique values for SibSp is', len(train.SibSp.unique()))
print('Number of Unique values for Embarked is', len(train.Embarked.unique()))
print('Number of Unique values for Cabin is', len(train.Cabin.unique()))
print('Number of Unique values for Sex is', len(train.Sex.unique()))
print('Number of Unique values for Ticket is', len(train.Ticket.unique()))
def cat_mean_encoding(df, column):
cumulative_sum = df.groupby(column)["AdoptionSpeed"].cumsum() - df["AdoptionSpeed"]
cumulative_count = df.groupby(column ).cumcount()
df[column + "_mean_encoding"] = cumulative_sum/cumulative_count
df[column + "_mean_encoding"].fillna(df.AdoptionSpeed.mean() , inplace=True )<categorify>
train.drop(["PassengerId"],inplace=True,axis=1)
 Test.drop(["PassengerId"],inplace=True,axis=1 )
class KFoldTargetEncoderTrain(base.BaseEstimator, base.TransformerMixin):
def __init__(self, colnames,targetName,n_fold=5,verbosity=True,discardOriginal_col=False):
self.colnames = colnames
self.targetName = targetName
self.n_fold = n_fold
self.verbosity = verbosity
self.discardOriginal_col = discardOriginal_col
def fit(self, X, y=None):
return self
def transform(self,X):
assert(type(self.targetName)== str)
 assert(type(self.colnames)== str)
 assert(self.colnames in X.columns)
 assert(self.targetName in X.columns)
 mean_of_target = X[self.targetName].mean()
kf = StratifiedKFold(n_splits=self.n_fold, shuffle=True)
 col_mean_name = self.colnames + '_' + 'Kfold_Target_Enc'
X[col_mean_name] = np.nan
for tr_ind, val_ind in kf.split(X,X[self.targetName]):
X_tr, X_val = X.iloc[tr_ind], X.iloc[val_ind]
X.loc[X.index[val_ind], col_mean_name] = X_val[self.colnames].map(X_tr.groupby(self.colnames)[self.targetName].mean())
 X[col_mean_name].fillna(mean_of_target, inplace = True)
 if self.verbosity:
encoded_feature = X[col_mean_name].values
print('Correlation between the new feature, {} and, {} is {}.'.format(col_mean_name,
self.targetName,
np.corrcoef(X[self.targetName].values, encoded_feature)[0][1]))
if self.discardOriginal_col:
X = X.drop(self.targetName, axis=1)
 return X
class KFoldTargetEncoderTest(base.BaseEstimator, base.TransformerMixin):
def __init__(self,train,colNames,encodedName):
self.train = train
self.colNames = colNames
self.encodedName = encodedName
def fit(self, X, y=None):
return self
def transform(self,X):
mean = self.train[[self.colNames,self.encodedName]].groupby(self.colNames ).mean().reset_index()
dd = {}
for index, row in mean.iterrows() :
dd[row[self.colNames]] = row[self.encodedName]
X[self.encodedName] = X[self.colNames]
X[self.encodedName] = X[self.encodedName].map(dd)
 return X<categorify>
train.drop(["Ticket"],inplace=True,axis=1)
 Test.drop(["Ticket"],inplace=True,axis=1 )
targetc = KFoldTargetEncoderTrain('Breed1','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 targetc = KFoldTargetEncoderTrain('Breed2','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Breed1','Breed1_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 test_targetc = KFoldTargetEncoderTest(train,'Breed2','Breed2_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
<categorify>
def name_sep(data):
families=[]
titles = []
new_name = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(",")[0]
title = name_no_bracket.split(",")[1].strip().split(" ")[0]
for c in string.punctuation:
name = name.replace(c,"" ).strip()
family = family.replace(c,"" ).strip()
title = title.replace(c,"" ).strip()
families.append(family)
 titles.append(title)
 new_name.append(name)
 return families, titles, new_name
cat_mean_encoding(train, 'Breed1')
 cat_mean_encoding(train, 'Breed2')
 test_targetc = KFoldTargetEncoderTest(train,'Breed1','Breed1_mean_encoding')
 test= test_targetc.fit_transform(test)
 test_targetc = KFoldTargetEncoderTest(train,'Breed2','Breed2_mean_encoding')
 test= test_targetc.fit_transform(test )<count_missing_values>
train['Surname'], train['Title'], train['Newname'] = name_sep(train.Name)
 Test['Surname'], Test['Title'], Test['Newname'] = name_sep(Test.Name)
 train.head()
train.Title.value_counts()
print(train.Breed1_Kfold_Target_Enc.isna().sum())
 print(train.Breed2_Kfold_Target_Enc.isna().sum())
 print(test.Breed1_Kfold_Target_Enc.isna().sum())
 print(test.Breed2_Kfold_Target_Enc.isna().sum())
 print(train.Breed1_mean_encoding.isna().sum())
 print(train.Breed2_mean_encoding.isna().sum())
 print(test.Breed1_mean_encoding.isna().sum())
 print(test.Breed2_mean_encoding.isna().sum())
 test[~test.Breed1.isin(train.Breed1)].head(5 )<categorify>
train['Title'] = train['Title'].replace(['Ms', 'Mlle'],'Miss')
 train['Title'] = train['Title'].replace(['Mme'],'Mrs')
 train['Title'] = train['Title'].replace(['Dr','Rev','the','Jonkheer','Lady','Sir', 'Don'],'Nobles')
 train['Title'] = train['Title'].replace(['Major','Col', 'Capt'],'Navy')
 train.Title.value_counts()
test.Breed1_Kfold_Target_Enc.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed2_Kfold_Target_Enc.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed1_mean_encoding.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed2_mean_encoding.fillna(train.AdoptionSpeed.mean() , inplace=True )<train_model>
nobles_survival = train[train.Title == "Nobles"].groupby(['Sex'] ).Survived.value_counts()
nobles_survival
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding'] 
<categorify>
Test.Title.value_counts()
Test['Title'] = Test['Title'].replace(['Ms','Dona'],'Miss')
 Test['Title'] = Test['Title'].replace(['Dr','Rev'],'Nobles')
 Test['Title'] = Test['Title'].replace(['Col'],'Navy')
 Test.Title.value_counts()
rescuer_count = train.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count_dict = rescuer_count.set_index('RescuerID' ).T.to_dict('list')
 train.RescuerID = train.RescuerID.apply(lambda x: np.log(rescuer_count_dict.get(x)[0]))
binner = KBinsDiscretizer(n_bins=6,encode='ordinal', strategy='kmeans')
 train['rescuer_bin_kmeans'] = pd.DataFrame(binner.fit_transform(train[['RescuerID']].copy()))
targetc = KFoldTargetEncoderTrain('rescuer_bin_kmeans','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 rescuer_count = test.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count_dict = rescuer_count.set_index('RescuerID' ).T.to_dict('list')
 test.RescuerID = test.RescuerID.apply(lambda x: np.log(rescuer_count_dict.get(x)[0]))
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='kmeans')
 test['rescuer_bin_kmeans'] = pd.DataFrame(binner.fit_transform(test[['RescuerID']].copy()))
test_targetc = KFoldTargetEncoderTest(train,'rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )<drop_column>
print('Missing values in Train set:', train.Cabin.isnull().sum())
 print('Missing values in Test set:', Test.Cabin.isnull().sum() )
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'] 
<categorify>
train.Cabin.value_counts()
targetc = KFoldTargetEncoderTrain('State','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'State','State_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )<define_variables>
def cabin_sep(data_cabin):
cabin_type = []
for i in range(len(data_cabin)) :
if data_cabin.isnull() [i] == True:
cabin_type.append('M')
 else:
cabin = data_cabin[i]
cabin_type.append(cabin[:1])
 return cabin_type
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
] 
<categorify>
train['Cabin'] = train['Cabin'].fillna('M' ).astype(str ).apply(lambda cabin: cabin[0] )
targetc = KFoldTargetEncoderTrain('Color1','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color1','Color1_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 targetc = KFoldTargetEncoderTrain('Color2','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color2','Color2_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 targetc = KFoldTargetEncoderTrain('Color3','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color3','Color3_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )<define_variables>
idx = train[train['Cabin'] == 'T'].index
train.loc[idx, 'Cabin'] = 'A'
train.Cabin.value_counts()
Test['Cabin'] = Test['Cabin'].fillna('M' ).astype(str ).apply(lambda cabin: cabin[0])
 Test.Cabin.value_counts()
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
] 
<feature_engineering>
train_categorical_features = ['Pclass', 'Sex','Title','Cabin', 'Embarked']
for feature in train_categorical_features:
dummies = pd.get_dummies(train[feature] ).add_prefix(feature + '_')
 train = train.join(dummies)
 train
train.Description.fillna('none', inplace=True)
 test.Description.fillna('none', inplace=True)
 train['desc_length'] = train.Description.apply(len)
 test['desc_length'] = test.Description.apply(len )<concatenate>
test_categorical_features = ['Pclass', 'Sex','Title', 'Cabin', 'Embarked']
for feature in test_categorical_features:
dummies = pd.get_dummies(Test[feature] ).add_prefix(feature + '_')
 Test = Test.join(dummies)
 Test
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 4), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(n_components=5)
 train_test = pd.concat([train, test], ignore_index=True, sort=False)
 tfidf_col = tfv.fit_transform(train_test.Description)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_')
 pcas = pd.concat([train_test, svd_col], axis=1)
 train = pcas.loc[np.isfinite(pcas.AdoptionSpeed), :]
test = pcas.loc[~np.isfinite(pcas.AdoptionSpeed), :]<define_variables>
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectKBest
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import learning_curve,GridSearchCV
from sklearn.ensemble.gradient_boosting import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
import statsmodels.formula.api as sm
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC, LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import roc_curve, auc
import scikitplot as skplt
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_absolute_error, accuracy_score
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ] 
<define_variables>
drop_column = ['Pclass','Name','Sex','Cabin', 'Embarked','Surname','Title','Newname']
train.drop(drop_column, axis=1, inplace = True)
 drop_column = ['Pclass','Name','Sex','Cabin', 'Embarked','Surname','Title','Newname']
Test.drop(drop_column, axis=1, inplace = True )
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
]
features = train[useColumns]
kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 for alg in []:
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 rfecv = RFECV(estimator=alg, step=1, cv=kf, scoring=kappa_score, n_jobs=-1)
 rfecv.fit(features,train['AdoptionSpeed'])
 print("Optimal number of features : %d" % rfecv.n_features_)
 print(rfecv.support_)
 print(features.columns[rfecv.support_])
 plt.figure()
plt.xlabel("Number of features selected")
 plt.ylabel("Cross validation score(nb of correct classifications)")
 plt.plot(range(1, len(rfecv.grid_scores_)+ 1), rfecv.grid_scores_)
 plt.show()<prepare_x_and_y>
original_train_set_without_survived = train.drop("Survived", axis=1)
 orginal_train_set_with_only_survived = train["Survived"]
useColumns = ['Breed1_Kfold_Target_Enc', 'Breed2_Kfold_Target_Enc', 'FurLength',
'Sterilized', 'State_Kfold_Target_Enc', 'PhotoAmt', 'age_bin',
'human_age', 'Breed1_mean_encoding', 'Breed2_mean_encoding',
'rescuer_bin_kmeans', 'rescuer_bin_kmeans_Kfold_Target_Enc']
params = {
'n_estimators': [50, 100, 200],
'learning_rate' : [0.01,0.05,0.1,0.3,1],
'base_estimator': [ensemble.AdaBoostClassifier() , ensemble.GradientBoostingClassifier() ]
}
X_train = train[useColumns]
y_train = train['AdoptionSpeed']
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 cv_split = kf.split(X_train, y_train)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 tuning = GridSearchCV(estimator = ensemble.AdaBoostClassifier() , param_grid = params,
scoring=kappa_score,cv=5, n_jobs=-1,iid=False)
<define_search_space>
x_train, x_test, y_train, y_test = train_test_split(
 original_train_set_without_survived, orginal_train_set_with_only_survived, train_size=0.8, test_size=0.2, random_state=0)
 print(x_train.shape, y_train.shape)
 print(x_test.shape, y_test.shape )
params = {'learning_rate':[0.15,0.1,0.05,0.01,0.005,0.001],
'n_estimators':[100,250,500,750,1000,1250,1500,1750],
'max_depth':[2,3,4,5,6,7],
'min_samples_split':[2,4,6,8,10,20,40,60,100],
'min_samples_leaf':[1,3,5,7,9],
'max_features':[2,3,4,5,6,7],
'subsample':[0.7,0.75,0.8,0.85,0.9,0.95,1]}
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 tuning = GridSearchCV(estimator = ensemble.GradientBoostingClassifier() , param_grid = params,
scoring=kappa_score,cv=kf, n_jobs=-1,iid=False)
 train_cleaned = train.drop(['Name','human_age','lifestage','RescuerID','Description','PetID', 'dataset_type'],axis=1)
 y_train = train_cleaned['AdoptionSpeed']
X_train = train_cleaned.drop(['AdoptionSpeed'],axis=1)
<import_modules>
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
 x_test = sc.transform(x_test)
 Test = sc.transform(Test )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<define_variables>
logreg = LogisticRegression()
logreg.fit(x_train,y_train)
 y_pred = logreg.predict(x_test)
 print("So, Our accuracy Score is: {}".format(round(accuracy_score(y_test,y_pred),8)) )
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt','has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ]
target = train.AdoptionSpeed.astype('int')
 X_train=train[useColumns]
X_test=test[useColumns] 
<define_variables>
def confusion_matrix_model(model_used):
cm=confusion_matrix(y_test,model_used.predict(x_test))
col=["Predicted Dead","Predicted Survived"]
cm=pd.DataFrame(cm)
 cm.columns=["Predicted Dead","Predicted Survived"]
cm.index=["Actual Dead","Actual Survived"]
cm[col]=np.around(cm[col].div(cm[col].sum(axis=1),axis=0),decimals=2)
 return cm
confusion_matrix_model(logreg )
useColumns = ['AdoptionSpeed','Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt','has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ]
target = train.AdoptionSpeed.astype('int')
 X_train=train[useColumns]
X_test=test[useColumns].drop(['AdoptionSpeed'], axis=1 )<init_hyperparams>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}<split>
x = original_train_set_without_survived
y = orginal_train_set_with_only_survived
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
x = sc.fit_transform(x )
model, oof_train, oof_test = run_xgb(xgb_params, X_train, X_test )<compute_train_metric>
def Acc_score(model):
return np.mean(cross_val_score(model,x,y,cv=k_fold,scoring="accuracy"))
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
logreg2=LogisticRegression()
print("Accuracy Scores: " + format(cross_val_score(logreg2,x,y,cv=k_fold,scoring="accuracy")))
 print(" ")
 print("Mean Accuracy Score: " + str(Acc_score(logreg2)))
 print(" ")
 print("Standard Deviation:", cross_val_score(logreg2,x,y,cv=k_fold,scoring="accuracy" ).std() )
coefficients_ = coefficients.copy()
coefficients_[0] = 1.65
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8 )<save_to_csv>
scores_auc = cross_val_score(logreg2, x, y, cv=k_fold, scoring='roc_auc')
 print("AUC score for 10 fold Cross Validation:", scores_auc)
 print(" ")
 print("Mean AUC score for 10 fold Cross Validation:", scores_auc.mean() )
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False )<set_options>
def run_k_fold(modeller, no_of_folds):
scores = []
mean_fpr = np.linspace(0, 1, 100)
 tprs = []
aucs = []
plt.figure(figsize =[10,9])
 k_fold = KFold(n_splits=no_of_folds, shuffle=True, random_state=0)
 fold = 1
for rounds in k_fold.split(x, y):
train_rows = rounds[0]
test_rows = rounds[1]
CV_x_train = x[train_rows]
CV_y_train = y[train_rows]
CV_x_test = x[test_rows]
CV_y_test = y[test_rows]
model = modeller()
model.fit(CV_x_train, CV_y_train)
 scores.append(model.score(CV_x_test, CV_y_test))
curr_fpr, curr_tpr, curr_auc = plt_roc_curve(
 'log reg fold ' + str(fold), model, CV_x_test, CV_y_test)
 tprs.append(np.interp(mean_fpr, curr_fpr, curr_tpr))
tprs[-1][0] = 0.0
roc_auc = auc(curr_fpr, curr_tpr)
 aucs.append(roc_auc)
 fold += 1
mean_tpr = np.mean(tprs, axis=0)
 mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
 std_auc = np.std(aucs)
 plt.plot(mean_fpr, mean_tpr, color='b',
label=r'Mean ROC(AUC = %0.8f $\pm$ %0.2f)' %(mean_auc, std_auc),
lw=2, alpha=.8)
 std_tpr = np.std(tprs, axis=0)
 tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
 tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
 plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
label=r'$\pm$ 1 std.dev.')
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Receiver operating characteristic example')
 plt.legend(loc="lower right")
 plt.show()
return np.mean(scores )
np.random.seed(369 )<compute_test_metric>
accuracy_score = run_k_fold(LogisticRegression, 10)
 print("Accuracy Score:", accuracy_score)
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
max_depth = range(1,30)
 max_feature = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,'auto']
criterion=["entropy", "gini"]
param = {'max_depth':max_depth,
'max_features':max_feature,
'criterion': criterion}
grid = GridSearchCV(DecisionTreeClassifier(random_state=42),
param_grid = param,
verbose=False,
cv=KFold(n_splits=10, shuffle=True, random_state=42),
n_jobs = -1)
 grid.fit(x, y)
 print(grid.best_params_)
 print(" ")
 print(grid.best_score_)
 print(" ")
 print(grid.best_estimator_)
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_csv>
dectree_grid = grid.best_estimator_
dectree_grid.score(x,y )
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16<define_variables>
rf_model = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',
max_depth=9, max_features='auto', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=16,
min_weight_fraction_leaf=0.0, n_estimators=140,
n_jobs=None, oob_score=False, random_state=42, verbose=0,
warm_start=True)
 rf_model.fit(x,y)
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1<define_search_model>
rf_model.score(x,y )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<predict_on_test>
Test = sc.fit_transform(Test)
 Test1 = pd.read_csv(".. /input/titanic/test.csv")
 output3 = pd.DataFrame({"PassengerId": Test1.PassengerId, "Survived":rf_model.predict(Test)})
 output3.PassengerId = output3.PassengerId.astype(int)
 output3.Survived = output3.Survived.astype(int)
 output3.to_csv("output3.csv", index=False)
 print("Your submission was successfully saved!")
 output3.head(10 )