Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
warnings.filterwarnings('ignore' )
submission.to_csv('submission.csv', index=False )<import_modules>
df_test = pd.read_csv(".. /input/titanic/test.csv")
 df_train = pd.read_csv(".. /input/titanic/train.csv")
 def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
import pandas as pd
import numpy as np
import json
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression
import statsmodels.api as sm
from sklearn import metrics
import hypertools as hyp
from imblearn.over_sampling import SMOTE
from xgboost import XGBClassifier, DMatrix, cv
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RandomizedSearchCV<load_from_csv>
total_missing_train = df_train.isnull().sum().sort_values(ascending=False)
 percent_1 = df_train.isnull().sum() /df_train.isnull().count() *100
percent_2 =(round(percent_1, 1)).sort_values(ascending=False)
 train_missing_data = pd.concat([total_missing_train, percent_2], axis=1, keys=['Total', '%'])
 print(total_missing_train)
 print('_'*25)
 train_missing_data.head(5 )
pets = pd.read_csv('.. /input/train/train.csv')
 pets2 = pd.read_csv('.. /input/test/test.csv' )<define_variables>
total_missing_test = df_test.isnull().sum().sort_values(ascending=False)
 percent_3 = df_test.isnull().sum() /df_test.isnull().count() *100
percent_4 =(round(percent_3, 1)).sort_values(ascending=False)
 test_missing_data = pd.concat([total_missing_test, percent_4], axis=1, keys=['Total', '%'])
 print(total_missing_test)
 print('_'*25)
 test_missing_data.head(5 )
petID = np.asarray(pets2.PetID )<feature_engineering>
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {} '.format(pclass, sex, age_by_pclass_sex[sex][pclass].astype(int)))
 df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
def add_pure(dataframe):
dataframe['pure_bred'] = 0
for i in range(0,(len(pets))):
try:
if dataframe.Breed2[i] == 0 or(dataframe.Breed1[i] == dataframe.Breed2[i]):
dataframe.iat[i, dataframe.columns.get_loc('pure_bred')] = 1
except:
continue
return(dataframe)
 def add_sentiments(dataframe, path):
dataframe['sentiment_score'] = float(0)
 dataframe['sentiment_magnitude'] = float(0)
 for i in range(0,(len(dataframe))):
filename = path + dataframe.PetID[i] + '.json'
try:
with open(filename, 'r')as f:
sentiments = json.load(f)
 dataframe.iat[i, dataframe.columns.get_loc('sentiment_score')] = sentiments['documentSentiment']['score']
dataframe.iat[i, dataframe.columns.get_loc('sentiment_magnitude')] = sentiments['documentSentiment']['magnitude']
except FileNotFoundError:
continue
return(dataframe)
 def create_cats(dataframe):
for i in dataframe.columns:
if i not in ['Age', 'AdoptionSpeed', 'FurLength', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'sentiment_score', 'sentiment_magnitude', 'desc_len', 'vertex_x', 'vertex_y','bounding_confidence', 'bounding_importance','dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac', 'dominant_score', 'label_score']:
dataframe[i] = dataframe[i].astype('category')
 def sentiment_to_float(dataframe):
dataframe.sentiment_score = dataframe.sentiment_score.astype(float)
 dataframe.sentiment_magnitude = dataframe.sentiment_magnitude.astype(float )<define_variables>
df_all[df_all['Embarked'].isnull() ]
def image_data(dataframe, ext):
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in dataframe.PetID:
try:
with open(ext + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 dataframe.loc[:, 'vertex_x'] = vertex_xs
dataframe.loc[:, 'vertex_y'] = vertex_ys
dataframe.loc[:, 'bounding_confidence'] = bounding_confidences
dataframe.loc[:, 'bounding_importance'] = bounding_importance_fracs
dataframe.loc[:, 'dominant_blue'] = dominant_blues
dataframe.loc[:, 'dominant_green'] = dominant_greens
dataframe.loc[:, 'dominant_red'] = dominant_reds
dataframe.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
dataframe.loc[:, 'dominant_score'] = dominant_scores
dataframe.loc[:, 'label_description'] = label_descriptions
dataframe.loc[:, 'label_score'] = label_scores<create_dataframe>
df_all['Embarked'] = df_all['Embarked'].fillna('S')
def fillna(dataframe):
for i in dataframe.columns:
if i not in ['Name', 'Description', 'PetID', 'RescuerID']:
if i not in ['Age', 'Quantity', 'PhotoAmt', 'VideoAmt', 'sentiment_score', 'sentiment_magnitude', 'Fee']:
dataframe[i].fillna(dataframe[i].mode() [0], inplace = True)
 else:
dataframe[i].fillna(dataframe[i].mean() , inplace = True )<feature_engineering>
df_all[df_all['Fare'].isnull() ]
def add_length(dataframe):
dataframe['desc_len'] = 0
for i in range(0, len(dataframe)) :
dataframe.iat[i, dataframe.columns.get_loc('desc_len')] = len(str(dataframe.Description[i]))
def sqrt_age(dataframe):
dataframe.Age = np.sqrt(dataframe.Age )<define_variables>
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'])['Fare'].median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )
def find_weights(dataframe):
weights = []
count = 0
for i in range(0,5):
count = 0
weight = 0.0
for j in dataframe.AdoptionSpeed:
if j == i:
count += 1
weight = count/len(dataframe.AdoptionSpeed)
 weights.append(weight)
 return weights<categorify>
data1=df_train.copy()
data1['Family_size'] = data1['SibSp'] + data1['Parch'] +1
data1['Family_size'].value_counts().sort_values(ascending=False )
add_pure(pets)
 add_pure(pets2)
 add_sentiments(pets, '.. /input/train_sentiment/')
 add_sentiments(pets2, '.. /input/test_sentiment/')
 image_data(pets, '.. /input/train_metadata/')
 image_data(pets2, '.. /input/test_metadata/')
<categorify>
df_all['Deck'] = df_all['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 df_all_decks = df_all.groupby(['Deck', 'Pclass'] ).count().drop(columns=['Survived', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Embarked', 'Cabin', 'PassengerId',
'Ticket'] ).rename(columns={'Name': 'Count'})
 df_all_decks
create_cats(pets)
 create_cats(pets2)
 sentiment_to_float(pets)
 sentiment_to_float(pets2 )<categorify>
def get_pclass_dist(df):
deck_counts = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {}, 'G': {}, 'M': {}, 'T': {}}
decks = df.columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = df[deck][pclass][0]
deck_counts[deck][pclass] = count
except KeyError:
deck_counts[deck][pclass] = 0
df_decks = pd.DataFrame(deck_counts)
 deck_percentages = {}
for col in df_decks.columns:
deck_percentages[col] = [(count / df_decks[col].sum())* 100 for count in df_decks[col]]
return deck_counts, deck_percentages,df_decks
all_deck_count, all_deck_per,df_decks_return = get_pclass_dist(df_all_decks)
 print(df_decks_return)
 print("_"*25)
 all_deck_per
add_length(pets)
 add_length(pets2)
 pets = pets.drop(['Name', 'PetID', 'Description','RescuerID'], axis = 1)
 pets2 = pets2.drop(['Name', 'PetID', 'Description','RescuerID'], axis = 1)
 pets = pd.get_dummies(pets)
 pets2 = pd.get_dummies(pets2)
<feature_engineering>
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'
def mat_interactions(dataframe):
dataframe['mat_int_1_1'] = dataframe.MaturitySize_1 * dataframe.Type_1
dataframe['mat_int_1_2'] = dataframe.MaturitySize_2 * dataframe.Type_1
dataframe['mat_int_1_3'] = dataframe.MaturitySize_3 * dataframe.Type_1
dataframe['mat_int_1_4'] = dataframe.MaturitySize_4 * dataframe.Type_1
dataframe['mat_int_2_1'] = dataframe.MaturitySize_1 * dataframe.Type_2
dataframe['mat_int_2_2'] = dataframe.MaturitySize_2 * dataframe.Type_2
dataframe['mat_int_2_3'] = dataframe.MaturitySize_3 * dataframe.Type_2
dataframe['mat_int_2_4'] = dataframe.MaturitySize_4 * dataframe.Type_2<feature_engineering>
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()
def age_interactions(dataframe):
dataframe['age_int_1_1'] = dataframe.Age * dataframe.Health_1
dataframe['age_int_1_2'] = dataframe.Age * dataframe.Health_2
dataframe['age_int_1_3'] = dataframe.Age * dataframe.Health_3<define_variables>
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
print(df_test.isnull().sum())
 print('-'*25 )
pets_vars = pets.columns.tolist()
adopt = ['AdoptionSpeed']
X=[i for i in pets_vars if i not in adopt]<categorify>
df_all = concat_df(df_train, df_test)
 df_all.head()
k=find_weights(pets)
 balance_weight = {0 :int(( max(k)*len(pets)) -(min(k)*len(pets)))}<normalization>
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )
train_y = pets[adopt]
smote = SMOTE(sampling_strategy = 'not majority', k_neighbors = 100)
 new_XX, train_y = smote.fit_resample(pets[X], train_y.values.ravel())
<choose_model_class>
df_all['Age'] = pd.qcut(df_all['Age'], 10 )
model = ExtraTreesClassifier()
model.fit(new_XX, train_y)
 feature_list = {}
for i in range(0, len(model.feature_importances_)) :
feature_list.update({i:model.feature_importances_[i]})
 feature_list = sorted(feature_list.items() , key=lambda kv: kv[1], reverse = True )<define_variables>
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )
j = feature_list[0:50]
new_X = []
for i in j:
new_X.append(X[i[0]])
 new_X<create_dataframe>
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1
train_X = pd.DataFrame(data = new_XX, columns=X)
 train_x = train_X[new_X]
test_x = pets2[new_X]<train_model>
df_train,df_test= divide_df(df_all)
 dfs=[df_train,df_test]
rf_model = RandomForestClassifier(class_weight = 'balanced', n_estimators = 1000, random_state = 42)
 rf_model.fit(train_x, train_y )<choose_model_class>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA
parameters = {'learning_rate':0.15, 'max_depth':6,'subsample':0.5,'objective':'multi:softmax', 'num_class':5}
model = XGBClassifier(**parameters )<count_unique_values>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2
np.unique(train_y, return_counts = True )<train_model>
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )
model.fit(train_x,train_y, eval_metric = 'merror', verbose = True)
 pred_xg = model.predict(test_x)
 pred_xg = np.round(pred_xg)
 pred_xg = pred_xg.astype(int )<data_type_conversions>
onehot_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in onehot_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )
df = {'PetID': petID, 'AdoptionSpeed': pred_xg}
df = pd.DataFrame(df)
 df.AdoptionSpeed = df.AdoptionSpeed.astype('int32' )<count_values>
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()
df.AdoptionSpeed.value_counts(dropna= False )<save_to_csv>
X = df_train.drop(columns=drop_cols )
df.to_csv('submission.csv', index = False )<define_variables>
X_train = StandardScaler().fit_transform(X)
 Y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('Y_train shape: {}'.format(Y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
BALANCING = False
MODEL_USE = 3 
<compute_test_metric>
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 sgd.score(X_train, Y_train)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2 )
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic' )<load_from_csv>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2 )
breeds = pd.read_csv('.. /input/breed_labels.csv')
 colors = pd.read_csv('.. /input/color_labels.csv')
 train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sub = pd.read_csv('.. /input/test/sample_submission.csv')
 states = pd.read_csv('.. /input/state_labels.csv')
<set_options>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2 )
%matplotlib inline
plt.style.use('ggplot' )<count_values>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2 )
train['AdoptionSpeed'].value_counts()<count_values>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2 )
states_to_ID = states.set_index('StateName')
 state_value_counts = train['State'].value_counts(ascending=False)
 state_distribution = states_to_ID['StateID'].map(state_value_counts ).sort_values(ascending=False)
 state_distribution 
<feature_engineering>
perceptron = Perceptron(max_iter=5)
 perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2 )
train['State'] = train['State'].replace(41401, 41326 )<feature_engineering>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2 )
train['Description'] = train['Description'].fillna('')
 test['Description'] = test['Description'].fillna('')
 train['desc_len'] = train['Description'].apply(lambda x: len(x))<prepare_x_and_y>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2 )
target_train = train['AdoptionSpeed']
cleaned_train = train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 test_pet_ID = test['PetID']
test_X = test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID'])
<count_missing_values>
rf = RandomForestClassifier(n_estimators=100,oob_score=True)
 scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy")
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )
target_train.isnull().values.any()<count_missing_values>
rf.fit(X_train, Y_train)
 Y_prediction = rf.predict(X_test)
 rf.score(X_train, Y_train)
 acc_random_forest = round(rf.score(X_train, Y_train)* 100, 2)
 print(round(acc_random_forest,2,), "%" )
test_X.isnull().values.any()<define_variables>
random_forest = RandomForestClassifier(n_estimators=100, oob_score = True)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 print(round(acc_random_forest,2,), "%" )
seed = 42<choose_model_class>
print("oob score:", round(random_forest.oob_score_, 4)*100, "%" )
class EnsembleModel:
def __init__(self,balancing=False):
self.balance_ratio = 5 if balancing else 1
self.rf_model = RandomForestClassifier()
self.lgb_model = lgb.LGBMClassifier()
self.rand_forest_params= {
'bootstrap': [True, False],
'max_depth': [30,50],
'min_samples_leaf': [20, 30],
'min_samples_split': [10,15],
'n_estimators': [250,300],
'random_state' : [seed]
}
self.lgb_params = {'objective' : ['multi:softprob'],
'eta' : [0.01],
'max_depth' : [7,8],
'num_class' : [5],
'num_leaves':[50],
'lambda' : [0.75],
'reg_alpha':[1e-5, 1e-2],
'silent': [1]
}
self.rf_best_param = None
self.lgb_best_param = None
self.columns = None
def set_scorer(self,kappa):
self.kappa = kappa
self.scorer = make_scorer(kappa)
 def set_param(self,rf_param,lgb_param):
self.rf_best_param = rf_param
self.lgb_best_param = lgb_param
def tune_best_param(self,x_train,y_train):
weights_train = [self.balance_ratio if i==0 else 1 for i in y_train.tolist() ]
rf_gridsearch = GridSearchCV(estimator = self.rf_model,
param_grid = self.rand_forest_params,
cv = 5,
n_jobs = -1,
verbose = 1,
scoring=self.scorer)
 rf_gridsearch.fit(x_train, y_train, sample_weight = weights_train)
 print('tuning for rf finished')
 self.rf_model = rf_gridsearch.best_estimator_
self.rf_best_param = rf_gridsearch.best_params_
lgb_gridsearch = GridSearchCV(self.lgb_model, self.lgb_params, n_jobs=-1,
cv=5,
scoring=self.scorer,
verbose=1, refit=True)
 lgb_gridsearch.fit(x_train, y_train, sample_weight = weights_train)
 print('tuning for lgb finished')
 self.lgb_model = lgb_gridsearch.best_estimator_
self.lgb_best_param = lgb_gridsearch.best_params_
print('best param for rf is:')
 print(self.rf_best_param)
 print('best param for lgb is:')
 print(self.lgb_best_param)
 def _avg(self,y_1,y_2):
return np.rint(( y_1 + y_2)/2.0 ).astype(int)
 def re_fit_with_best_param(self,X,y):
if self.rf_best_param == None or self.lgb_best_param == None:
print('use tune_best_param() method to get best param first')
 return
weights_train = [self.balance_ratio if i==0 else 1 for i in y.tolist() ]
self.rf_model = RandomForestClassifier()
self.lgb_model = lgb.LGBMClassifier()
self.rf_model.set_params(**self.rf_best_param)
 self.lgb_model.set_params(**self.lgb_best_param)
 self.rf_model.fit(X,y,sample_weight=weights_train)
 self.lgb_model.fit(X,y,sample_weight=weights_train)
 print('refit finished')
 def validate(self,x_valid, y_valid):
rf_score = self.kappa(self.rf_model.predict(x_valid), y_valid)
 print('{} score: {}'.format('rf', round(rf_score, 4)))
 lgb_score = self.kappa(self.lgb_model.predict(x_valid), y_valid)
 print('{} score: {}'.format('lgb', round(lgb_score, 4)))
 score = kappa(self._avg(self.lgb_model.predict(x_valid), self.rf_model.predict(x_valid)) , y_valid)
 print('{} score on validation set: {}'.format('combiner', round(score, 4)))
 self.columns = x_valid.columns
def predict(self,test_X):
rf_result = self.rf_model.predict(test_X)
 lgb_result = self.lgb_model.predict(test_X)
 final_result = self._avg(rf_result,lgb_result)
 return final_result
def get_feature_importance(self):
rf_feature_importances = pd.DataFrame({'Feature':self.columns.tolist() ,'importance':self.rf_model.feature_importances_.tolist() })
 lgb_feature_importances = pd.DataFrame({'Feature':self.columns.tolist() ,'importance':self.lgb_model.feature_importances_.tolist() })
 overall_feature_importance = pd.merge(rf_feature_importances, lgb_feature_importances, on='Feature', how='outer')
 overall_feature_importance['avg_importance'] =(overall_feature_importance['importance_x'] + overall_feature_importance['importance_y'])/2
overall_feature_importance = overall_feature_importance.sort_values(by=['avg_importance'], ascending=False)
 return overall_feature_importance 
<choose_model_class>
print("oob score:", round(rf.oob_score_, 4)*100, "%" )
<prepare_x_and_y>
random_forest = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=42,
n_jobs=-1,
verbose=1)
 random_forest.fit(X_train, Y_train)
 Y_prediction =(random_forest.predict(X_test)).astype(int)
 random_forest.score(X_train, Y_train)
 print("oob score:", round(random_forest.oob_score_, 4)*100, "%" )
target_train = train['AdoptionSpeed']
cleaned_train = train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 test_pet_ID = test['PetID']
test_X = test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID'])
<split>
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, Y_train), 1):
print('Fold {}
'.format(fold))
random_forest.fit(X_train[trn_idx], Y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(Y_train[trn_idx], random_forest.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(Y_train[val_idx],random_forest.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = random_forest.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = random_forest.predict_proba(X_test)[:, 1]
oob += random_forest.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, random_forest.oob_score_))
print('Average OOB Score: {}'.format(oob))
x_train, x_valid, y_train, y_valid = train_test_split(cleaned_train,
target_train,
test_size=0.2,
random_state=seed)
<train_model>
rf = RandomForestClassifier(n_estimators=100,oob_score=True)
 scores = cross_val_score(random_forest, X_train, Y_train, cv=10, scoring = "accuracy")
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )
if MODEL_USE == 1 or MODEL_USE==0:
first_model = EnsembleModel(balancing=True)
 first_model.set_scorer(kappa)
 first_model.tune_best_param(x_train, y_train)
 first_model.validate(x_valid, y_valid )<load_pretrained>
predictions = cross_val_predict(random_forest, X_train, Y_train, cv=3)
 confusion_matrix(Y_train, predictions )
filename = os.listdir(".. /input/train_sentiment")[1]
filename = ".. /input/train_sentiment/"+filename
with open(filename, 'r')as f:
sentiment = json.load(f)
 sentiment<create_dataframe>
print("Precision:", precision_score(Y_train, predictions))
print("Recall:",recall_score(Y_train, predictions))
def load_desc_sentiment(path):
all_desc_sentiment_files = os.listdir(path)
 count_file = len(all_desc_sentiment_files)
 desc_sentiment_df = pd.DataFrame(columns=['PetID','desc_senti_magnitude','desc_senti_score'])
 current_file_index = 1
for filename in all_desc_sentiment_files:
with open(path+filename, 'r')as f:
sentiment_json = json.load(f)
 petID = filename.split('.')[0]
magnitude = sentiment_json['documentSentiment']['magnitude']
score = sentiment_json['documentSentiment']['score']
desc_sentiment_df = desc_sentiment_df.append({'PetID': petID, 'desc_senti_magnitude':magnitude,'desc_senti_score':score}, \
ignore_index=True)
 if current_file_index % 1000 == 0 or current_file_index == count_file :
print('current progress: %d file of %d loaded' %(current_file_index,count_file))
current_file_index += 1
return desc_sentiment_df<categorify>
f1_score(Y_train, predictions )
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(train['Description'])
 desc_X_train = tfv.transform(train['Description'])
 desc_X_test = tfv.transform(test['Description'])
 print(desc_X_train.shape)
 print(desc_X_test.shape )<split>
r_a_score = roc_auc_score(Y_train, y_scores)
 print("ROC-AUC-Score:", r_a_score )
train_desc_sentiment_df = load_desc_sentiment(".. /input/train_sentiment/")
 test_desc_sentiment_df = load_desc_sentiment(".. /input/test_sentiment/" )<feature_engineering>
submission = pd.DataFrame({
"PassengerId": df_test["PassengerId"],
"Survived": Y_prediction
})
 submission.to_csv('submission.csv', index=False )