Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<sort_values>
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'
train_df[['Pclass','Survived']].groupby('Pclass', as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
train_df[['Sex','Survived']].groupby('Sex', as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
mapping = {"yes": 1, "no": 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefa', 'edjefe']].describe()
train_df[['SibSp','Survived']].groupby('SibSp', as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
test['Target'] = np.nan
data = train.append(test, ignore_index = True )
train_df[['Parch','Survived']].groupby('Parch', as_index=False ).mean().sort_values(by='Survived', ascending=False )<define_variables>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
def detect_outliers(df,columns):
outlier_list=[]
for c in columns:
Q1 = np.percentile(df[c],25)
 Q3 = np.percentile(df[c],75)
 IQR = Q3-Q1
outlier_step = 1.5*IQR
indices = df[(df[c] < Q1-outlier_step)|(df[c] > Q3+outlier_step)].index
outlier_list.extend(indices)
 outlier_list_counter=Counter(outlier_list)
 final_outlier_list = [i for i,v in outlier_list_counter.items() if v>2]
return final_outlier_list<filter>
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]
final_outlier_list = detect_outliers(train_df, ['Age', 'Fare', 'SibSp', 'Parch'])
 train_df.loc[final_outlier_list]<drop_column>
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))
train_df = train_df.drop(final_outlier_list, axis=0 ).reset_index(drop=True )<concatenate>
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )
train_df_len = len(train_df)
 train_df = pd.concat([train_df,test_df], axis=0 ).reset_index(drop=True )<features_selection>
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
train_df.columns[train_df.isnull().any() ]<count_missing_values>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )
train_df.isnull().sum()<filter>
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )
train_df[train_df['Embarked'].isnull() == True]<count_values>
data['v18q1'] = data['v18q1'].fillna(0 )
train_df['Embarked'][train_df['Pclass']==1].value_counts()<feature_engineering>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()
train_df['Embarked'] = train_df['Embarked'].fillna('C' )<filter>
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
train_df[train_df['Embarked'].isnull() == True]<filter>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
train_df[train_df['Fare'].isnull() ==True]<feature_engineering>
id_ = ['Id', 'idhogar', 'Target']
train_df['Fare'] = train_df['Fare'].fillna(train_df['Fare'][train_df.Pclass==3].mean() )<filter>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
train_df[train_df['Fare'].isnull() ==True]<filter>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
train_df[train_df['Age'].isnull() ]<feature_engineering>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
train_df['Sex']=list(1 if i=='male' else 0 for i in train_df['Sex'] )<feature_engineering>
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )
train_df['Sex']=list('male' if i==1 else 'female' for i in train_df['Sex'] )<define_variables>
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape
index_nan_age = list(train_df[train_df['Age'].isnull() ].index )<feature_engineering>
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
for i in index_nan_age:
age_prediction = train_df['Age'][(( train_df['SibSp']==train_df.iloc[i]['SibSp'])&(train_df['Parch']==train_df.iloc[i]['Parch'])&(train_df['Pclass']==train_df.iloc[i]['Pclass'])) ].median()
age_med = train_df['Age'].median()
if not np.isnan(age_prediction):
train_df['Age'].iloc[i] = age_prediction
else:
train_df['Age'].iloc[i] = age_med<filter>
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )
train_df[train_df['Age'].isnull() ]<import_modules>
corr_matrix.loc[corr_matrix['coopele'].abs() > 0.9, corr_matrix['coopele'].abs() > 0.9]
from sklearn.tree import DecisionTreeRegressor<train_model>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
fare = np.array(train_df.Fare[:train_df_len] ).reshape(-1,1)
 survived = np.array(train_df.Survived[:train_df_len] ).reshape(-1,1)
 DecTree = DecisionTreeRegressor(max_leaf_nodes=150)
 DecTree.fit(fare, survived)
 train_df['Fare_class'] = DecTree.apply(np.array(train_df.Fare ).reshape(-1,1))<categorify>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )
train_df = pd.get_dummies(data=train_df, columns=['Fare_class'] )<drop_column>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
train_df.drop('Fare', axis=1, inplace=True )<data_type_conversions>
counts = pd.DataFrame(heads.groupby(['walls+roof+floor'])['Target'].value_counts(normalize = True)).rename(columns = {'Target': 'Normalized Count'} ).reset_index()
counts.head()
train_df['Age'] = train_df['Age'].astype('int' )<feature_engineering>
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
train_df['Age_based_survival'] = 0<feature_engineering>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
for i in range(train_df['Age'].max() +1):
df = train_df[train_df['Age']==i][train_df.Survived.isna() == False]
avg_surv_prob = df['Survived'].mean()
train_df.loc[train_df.Age==i,'Age_based_survival'] = avg_surv_prob<feature_engineering>
from scipy.stats import spearmanr
train_df['Age_based_survival'] = train_df['Age_based_survival'].fillna(0)
 train_df[train_df['Age_based_survival'].isnull() ]<drop_column>
corrs = pcorrs.merge(scorrs, on = 'feature')
 corrs['diff'] = corrs['pcorr'] - corrs['scorr']
corrs.sort_values('diff' ).head()
train_df.drop('Age', axis=1, inplace=True)
 train_df.head()<feature_engineering>
ind = ind.drop(columns = 'male' )
train_df['Title'] = [i.split('.')[0].split(',')[-1].strip() for i in train_df['Name']]
train_df['Title']<feature_engineering>
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
train_df['Title'] = train_df['Title'].replace(['Lady','the Countess','Capt','Col','Don','Dr','Major','Rev','Sir','Jonkheer','Dona'],'other' )<feature_engineering>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
train_df['Title'] = [0 if i=='Master' else 1 if i=='Miss' or i=='Ms' or i=='Mlle' or i=='Mrs' else 2 if i=='Mr' else 3 for i in train_df['Title']]<drop_column>
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
train_df.drop('Name', axis=1, inplace=True )<categorify>
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )
train_df = pd.get_dummies(data=train_df, columns=['Title'] )<feature_engineering>
final.groupby('female-head')['Target'].value_counts(normalize=True )
train_df['Fsize'] = train_df['SibSp'] + train_df['Parch'] + 1<feature_engineering>
final.groupby('female-head')['meaneduc'].agg(['mean', 'count'] )
train_df['Family_size'] = [1 if i in [2,3,4] else 0 for i in train_df['Fsize']]<categorify>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
train_df['SibSp_categ'] = [0 if i in [0,1,2] else 1 if i in [3,4] else 2 for i in train_df.SibSp]<categorify>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
train_df = pd.get_dummies(data=train_df, columns=['SibSp_categ'] )<categorify>
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )
train_df['Parch_categ'] = [0 if i in [1,2,3] else 1 if i in [0,5] else 2 for i in train_df.Parch]<categorify>
model = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
train_df = pd.get_dummies(data=train_df, columns=['Parch_categ'] )<categorify>
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature': features, 'importance': model.feature_importances_})
 feature_importances.head()
train_df = pd.get_dummies(data=train_df, columns=['Embarked'] )<feature_engineering>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
ticket_list=[]
for i in train_df['Ticket']:
if not i.strip().isdigit() :
ticket_list.append(i.strip().replace('.','' ).replace('/','' ).split() [0])
 else:
ticket_list.append('x')
 train_df['Ticket'] = ticket_list<categorify>
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )
train_df = pd.get_dummies(train_df, columns=['Ticket'], prefix='T' )<categorify>
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )
train_df['Pclass'] = train_df['Pclass'].astype('category')
 train_df = pd.get_dummies(data=train_df, columns=['Pclass'])
 train_df.head()<categorify>
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )
train_df['Sex'] = train_df['Sex'].astype('category')
 train_df = pd.get_dummies(data=train_df, columns=['Sex'])
 train_df.head()<drop_column>
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )
train_df.drop(labels=['PassengerId', 'Cabin'], axis=1, inplace=True)
 train_df.columns<import_modules>
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score<drop_column>
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
test = train_df[train_df_len:]
test.drop(labels=['Survived'], axis=1, inplace=True)
 test.head()<split>
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )
train = train_df[:train_df_len]
y_train = train['Survived']
x_train = train.drop('Survived', axis=1)
 x_train,x_test,y_train,y_test = train_test_split(x_train,y_train,test_size=0.2, random_state=42)
 print('x_train: ',len(x_train))
print('x_test: ',len(x_test))
print('y_train: ',len(y_train))
print('y_test: ',len(y_test))
print('test: ',len(test))<train_model>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state=10),
'RF', model_results )
log_reg = LogisticRegression()
log_reg.fit(x_train,y_train)
 acc_logreg_train = round(log_reg.score(x_train,y_train)*100,2)
 acc_logreg_test = round(log_reg.score(x_test,y_test)*100,2)
 print('Training accuracy: %{}'.format(acc_logreg_train))
print('Testing accuracy: %{}'.format(acc_logreg_test))<choose_model_class>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
random_state = 42
model_list = [DecisionTreeClassifier(random_state = random_state),
SVC(random_state = random_state, probability=True),
RandomForestClassifier(random_state = random_state),
LogisticRegression(random_state = random_state),
KNeighborsClassifier() ]
dt_param_grid = {"min_samples_split" : range(10,500,20),
"max_depth": range(1,20,2)}
svc_param_grid = {"kernel" : ["rbf"],
"gamma": [0.001, 0.01, 0.1, 1],
"C": [1,10,50,100,200,300,1000]}
rf_param_grid = {"max_features": [1,3,10],
"min_samples_split":[2,3,10],
"min_samples_leaf":[1,3,10],
"bootstrap":[False],
"n_estimators":[100,300],
"criterion":["gini"]}
logreg_param_grid = {"C":np.logspace(-3,3,7),
"penalty": ["l1","l2"]}
knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int ).tolist() ,
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
grid_list = [dt_param_grid, svc_param_grid, rf_param_grid, logreg_param_grid, knn_param_grid]<train_on_grid>
def submit(model, train, train_labels, test, test_ids):

 model.fit(train, train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar': test_ids,
'Target': predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
score_list = []
best_estimators = []
for i in range(5):
my_model = GridSearchCV(estimator=model_list[i], param_grid=grid_list[i], scoring='accuracy', n_jobs=-1, cv=StratifiedKFold(n_splits=10))
my_model.fit(x_train, y_train)
 score_list.append(accuracy_score(my_model.best_estimator_.predict(x_test), y_test))
best_estimators.append(my_model.best_estimator_)
 print('Best scores: ', score_list, '
' )<train_model>
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 rf_submission.to_csv('rf_submission.csv', index = False )
votingC = VotingClassifier([('RandomForest',best_estimators[2]), 
('SVC',best_estimators[1]), 
('Logistic',best_estimators[3])], voting='soft', n_jobs=-1)
 votingC.fit(x_train, y_train)
 print('Ensembled score: ', accuracy_score(votingC.predict(x_test),y_test))<save_to_csv>
train_set = pd.DataFrame(train_set, columns = features)
 corr_matrix = train_set.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop
test_survived = pd.Series(votingC.predict(test), name='Survived' ).astype(int)
 results = pd.concat([test_PassengerId, test_survived], axis=1)
 results.to_csv('titanic.csv', index=False )<load_from_csv>
test_set = pd.DataFrame(test_set, columns = features)
 train_set, test_set = train_set.align(test_set, axis = 1, join = 'inner')
 features = list(train_set.columns )
train_data = pd.read_csv(".. /input/titanic/train.csv")
 test_data = pd.read_csv(".. /input/titanic/test.csv")
 train_data.columns<feature_engineering>
estimator = RandomForestClassifier(random_state = 10, n_estimators = 100, n_jobs = -1)
 selector = RFECV(estimator, step = 1, cv = 3, scoring= scorer, n_jobs = -1 )
def outlier_detect(feature, data):
outlier_index = []
for each in feature:
Q1 = np.percentile(data[each], 25)
 Q3 = np.percentile(data[each], 75)
 IQR = Q3 - Q1
min_quartile = Q1 - 1.5*IQR
max_quartile = Q3 + 1.5*IQR
outlier_list = data[(data[each] < min_quartile)|(data[each] > max_quartile)].index
outlier_index.extend(outlier_list)
 outlier_index = Counter(outlier_index)
 outlier_data = list(i for i, n in outlier_index.items() if n > 3)
 return outlier_data<filter>
selector.fit(train_set, train_labels )
outlier_data = outlier_detect(["Age","SibSp","Parch","Fare"], train_data)
 train_data.loc[outlier_data] 
<drop_column>
rankings = pd.DataFrame({'feature': list(train_set.columns), 'rank': list(selector.ranking_)} ).sort_values('rank')
 rankings.head(10 )
train_data = train_data.drop(outlier_data, axis=0 ).reset_index(drop=True )<concatenate>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
data = pd.concat([train_data, test_data], axis=0 ).reset_index(drop=True )<groupby>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )
data[["Sex", "Survived"]].groupby(["Sex"], as_index = False ).mean()<filter>
model_results = cv_model(train_selected, train_labels, model, 'RF-SEL', model_results )
data.columns[data.isnull().any() ]<count_missing_values>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
data.isnull().sum()<count_missing_values>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
data.isnull().sum()<filter>
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set, train_labels, test_set, test_ids, return_preds=True )
data[data["Fare"].isnull() ]<feature_engineering>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )
data["Fare"] = data["Fare"].fillna(np.mean(data[(( data["Pclass"]==3)&(data["Embarked"]==0)) ]["Fare"]))
data[data["Fare"].isnull() ]<filter>
%%capture --no-display
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids )
data[data["Embarked"].isnull() ]<feature_engineering>
model_results = model_results.append(pd.DataFrame({'model': ["GBM", "GBM_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )
data["Embarked"] = data["Embarked"].fillna(1)
 data[data["Embarked"].isnull() ]<filter>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )
data[data["Age"].isnull() ]<feature_engineering>
submission.to_csv('gbm_10fold.csv', index = False )
data_age_nan_index = data[data["Age"].isnull() ].index
for i in data_age_nan_index:
mean_age = data["Age"][(data["Pclass"]==data.iloc[i]["Pclass"])].median()
data["Age"].iloc[i] = mean_age<categorify>
%%capture
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )
data["Alone"] = [1 if i == 0 else 0 for i in data["Family"]]
data["Family"].replace([0,1,2,3,4,5,6,7,10], [0,1,1,1,0,2,0,2,2], inplace=True)
 data.head()<feature_engineering>
submission.to_csv('gmb_10fold_selected.csv', index = False )
data['Title']=data.Name.str.extract('([A-Za-z]+)\.' )<categorify>
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )
data['Title'].replace(['Mme','Ms','Mlle','Lady','Countess','Dona','Dr','Major','Sir','Capt','Don','Rev','Col', 'Jonkheer'],['Miss','Miss','Miss','Mrs','Mrs','Mrs','Mr','Mr','Mr','Mr','Mr','Other','Other','Other'], inplace=True )<categorify>
print(f"There are {gbm_fi_selected[gbm_fi_selected['importance'] == 0].shape[0]} features with no importance." )
data['Age_Limit'] = LabelEncoder().fit_transform(data['Age_Limit'])
<categorify>
from hyperopt import hp, tpe, Trials, fmin, STATUS_OK
from hyperopt.pyll.stochastic import sample
data['Fare_Limit'] = LabelEncoder().fit_transform(data['Fare_Limit'])
<data_type_conversions>
import csv
import ast
from timeit import default_timer as timer
data['Age']=data['Age'].astype(int)
 data.drop(labels=["SibSp","Parch","Cabin","Fare","Age", "Ticket", "Name", "PassengerId"], axis=1, inplace = True)
 data.head()<categorify>
def objective(hyperparameters, nfolds=5):

 global ITERATION
ITERATION += 1
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 subsample_freq = hyperparameters['boosting_type'].get('subsample_freq', 0)
 boosting_type = hyperparameters['boosting_type']['boosting_type']
if boosting_type == 'dart':
hyperparameters['drop_rate'] = hyperparameters['boosting_type']['drop_rate']
hyperparameters['subsample'] = subsample
hyperparameters['subsample_freq'] = subsample_freq
hyperparameters['boosting_type'] = boosting_type
if not hyperparameters['limit_max_depth']:
hyperparameters['max_depth'] = -1
for parameter_name in ['max_depth', 'num_leaves', 'subsample_for_bin',
'min_child_samples', 'subsample_freq']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 features = np.array(train_selected)
 labels = np.array(train_labels ).reshape(( -1))
valid_scores = []
best_estimators = []
run_times = []
model = lgb.LGBMClassifier(**hyperparameters, class_weight = 'balanced',
n_jobs=-1, metric = 'None',
n_estimators=10000)
 for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
start = timer()
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 400)
 end = timer()
valid_scores.append(model.best_score_['valid']['macro_f1'])
 best_estimators.append(model.best_iteration_)
 run_times.append(end - start)
 score = np.mean(valid_scores)
 score_std = np.std(valid_scores)
 loss = 1 - score
run_time = np.mean(run_times)
 run_time_std = np.std(run_times)
 estimators = int(np.mean(best_estimators))
hyperparameters['n_estimators'] = estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, score, score_std])
 of_connection.close()
if ITERATION % PROGRESS == 0:
display(f'Iteration: {ITERATION}, Current Score: {round(score, 4)}.')
 return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'time': run_time, 'time_std': run_time_std, 'status': STATUS_OK,
'score': score, 'score_std': score_std}
data = pd.get_dummies(data,columns=["Pclass"])
 data = pd.get_dummies(data,columns=["Embarked"])
 data = pd.get_dummies(data,columns=["Family"])
 data = pd.get_dummies(data,columns=["Age_Limit"])
 data = pd.get_dummies(data,columns=["Fare_Limit"])
 data = pd.get_dummies(data,columns=["Title"])
 data.head()<import_modules>
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt',
'subsample': hp.uniform('gdbt_subsample', 0.5, 1),
'subsample_freq': hp.quniform('gbdt_subsample_freq', 1, 10, 1)},
{'boosting_type': 'dart',
'subsample': hp.uniform('dart_subsample', 0.5, 1),
'subsample_freq': hp.quniform('dart_subsample_freq', 1, 10, 1),
'drop_rate': hp.uniform('dart_drop_rate', 0.1, 0.5)},
{'boosting_type': 'goss',
'subsample': 1.0,
'subsample_freq': 0}]),
'limit_max_depth': hp.choice('limit_max_depth', [True, False]),
'max_depth': hp.quniform('max_depth', 1, 40, 1),
'num_leaves': hp.quniform('num_leaves', 3, 50, 1),
'learning_rate': hp.loguniform('learning_rate',
np.log(0.025),
np.log(0.25)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 2000, 100000, 2000),
'min_child_samples': hp.quniform('min_child_samples', 5, 80, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.5, 1.0)
 }
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC<train_model>
algo = tpe.suggest
if len(data)==(len(train_data)+ len(test_data)) :
print("success" )<drop_column>
trials = Trials()
OUT_FILE = 'optimization.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 MAX_EVALS = 100
PROGRESS = 10
N_FOLDS = 5
ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score', 'std']
writer.writerow(headers)
 of_connection.close()
test = data[len(train_data):]
test.drop(labels="Survived", axis=1, inplace=True )<split>
%%capture --no-display
display("Running Optimization for {} Trials.".format(MAX_EVALS))
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS )
train = data[:len(train_data)]
X_train = train.drop(labels = "Survived", axis=1)
 y_train = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.3, random_state=42)
<train_model>
with open('trials.json', 'w')as f:
f.write(json.dumps(str(trials)) )
log_reg = LogisticRegression(random_state=42)
 log_reg.fit(X_train, y_train)
 print("Accuracy: ", log_reg.score(X_test,y_test))<train_model>
best_hyp = ast.literal_eval(results.loc[0, 'hyperparameters'])
 best_hyp
rf_reg = RandomForestClassifier(random_state=42)
 rf_reg.fit(X_train, y_train)
 print("Accuracy: ", rf_reg.score(X_test,y_test))<train_model>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_SEL"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )
svm_clsf = SVC()
svm_clsf.fit(X_train, y_train)
 print("Accuracy: ", svm_clsf.score(X_test,y_test))<train_model>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )
best_knn = []
for n in range(1,12):
knn = KNeighborsClassifier(n_neighbors=n)
 knn.fit(X_train, y_train)
 best_knn.insert(n, knn.score(X_test,y_test))
best_knn 
<train_model>
submission.to_csv('gbm_opt_10fold_selected.csv', index = False )
knn_clsf = KNeighborsClassifier(n_neighbors=8)
 knn_clsf.fit(X_train, y_train)
 print("Accuracy: ", knn_clsf.score(X_test,y_test))<train_model>
heads['Target'].value_counts()
voting_classfication = VotingClassifier(estimators = [('knn', knn_clsf),('lg', log_reg),('rfg', rf_reg),('svc', svm_clsf)], voting="hard", n_jobs=-1)
 voting_classfication.fit(X_train, y_train)
 print("Accuracy: ", voting_classfication.score(X_test,y_test))<save_to_csv>
preds['Target'].value_counts()
test_result = pd.Series(voting_classfication.predict(test), name = "Survived" ).astype(int)
 results = pd.concat([test_data["PassengerId"], test_result],axis = 1)
 results.to_csv("titanic_submission.csv", index = False )<set_options>
X_train, X_valid, y_train, y_valid = train_test_split(train_selected,
train_labels,
test_size = 1000,
random_state = 10)
 model = lgb.LGBMClassifier(**best_hyp,
class_weight = 'balanced',
random_state = 10)
 model.fit(X_train, y_train);
warnings.filterwarnings('ignore' )<load_from_csv>
valid_preds = model.predict_proba(X_valid)
 preds_df = pd.DataFrame(valid_preds, columns = [1, 2, 3, 4])
 preds_df['prediction'] = preds_df[[1, 2, 3, 4]].idxmax(axis = 1)
 preds_df['confidence'] = preds_df[[1, 2, 3, 4]].max(axis = 1)
 preds_df.head()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
print('F1 score:', round(f1_score(y_valid, preds_df['prediction'], average = 'macro'), 5))
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
n_components = 3
umap = UMAP(n_components=n_components)
 pca = PCA(n_components=n_components)
 ica = FastICA(n_components=n_components)
 tsne = TSNE(n_components=n_components )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<sort_values>
train_df = train_selected.copy()
test_df = test_selected.copy()
for method, name in zip([umap, pca, ica, tsne],
['umap', 'pca', 'ica', 'tsne']):
if name == 'tsne':
start = timer()
reduction = method.fit_transform(train_selected)
 end = timer()
else:
start = timer()
reduction = method.fit_transform(train_selected)
 end = timer()
test_reduction = method.transform(test_selected)
 test_df['%s_c1' % name] = test_reduction[:, 0]
test_df['%s_c2' % name] = test_reduction[:, 1]
test_df['%s_c3' % name] = test_reduction[:, 2]
train_df['%s_c1' % name] = reduction[:, 0]
train_df['%s_c2' % name] = reduction[:, 1]
train_df['%s_c3' % name] = reduction[:, 2]
print(f'Method: {name} {round(end - start, 2)} seconds elapsed.' )
train_targets_scored.sum() [1:].sort_values()<concatenate>
train_df, test_df = train_df.align(test_df, axis = 1, join = 'inner' )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_df, train_labels,
test_df, test_ids, nfolds = 10,
hyp = best_hyp )
n_comp = 15
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
submission.to_csv('gbm_opt_10fold_dr.csv', index = False )
var_thresh = VarianceThreshold(threshold=0.5)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features 
<merge>
model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_DR"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
model = RandomForestClassifier(max_depth = 3, n_estimators=10)
 model.fit(train_selected, train_labels)
 estimator_limited = model.estimators_[5]
estimator_limited
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<feature_engineering>
export_graphviz(estimator_limited, out_file='tree_limited.dot', feature_names = train_selected.columns,
class_names = ['extreme', 'moderate' , 'vulnerable', 'non-vulnerable'],
rounded = True, proportion = False, precision = 2, filled = True )
<feature_engineering>
!dot -Tpng tree_limited.dot -o tree_limited.png
<create_dataframe>
Image(filename = 'tree_limited.png' )
<drop_column>
model = RandomForestClassifier(max_depth = None, n_estimators=10)
 model.fit(train_selected, train_labels)
 estimator_nonlimited = model.estimators_[5]
export_graphviz(estimator_nonlimited, out_file='tree_nonlimited.dot', feature_names = train_selected.columns,
class_names = ['extreme', 'moderate' , 'vulnerable', 'non-vulnerable'],
rounded = True, proportion = False, precision = 2)
 !dot -Tpng tree_nonlimited.dot -o tree_nonlimited.png -Gdpi=600
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
Image(filename = 'tree_nonlimited.png' )