Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<save_to_csv>
%matplotlib inline
submission_df = pd.DataFrame(submission)
 submission_df.columns = submission_df.iloc[0]
submission_df = submission_df.drop(0, axis=0)
 submission_df.to_csv("submission.csv", index=False )<import_modules>
def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0)
 indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True )
import tensorflow as tf
from tensorflow import keras
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras import layers, Sequential, optimizers
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.utils import to_categorical
import matplotlib.pyplot as plt<load_from_csv>
data_path = ".. /input"
train = pd.read_csv(os.path.join(data_path, "train.csv"))
test = pd.read_csv(os.path.join(data_path, "test.csv"))
<load_from_csv>
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefa'] == "yes", "edjefa"] = 1
train['edjefa'] = train['edjefa'].astype("int")
 train.loc[train['edjefe'] == "no", "edjefe"] = 0
train.loc[train['edjefe'] == "yes", "edjefe"] = 1
train['edjefe'] = train['edjefe'].astype("int")
 test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test['edjefa'] = test['edjefa'].astype("int")
 test.loc[test['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
test['edjefe'] = test['edjefe'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0
raw_csv = "/kaggle/input/digit-recognizer/train.csv"
test_csv = "/kaggle/input/digit-recognizer/test.csv"
raw_df = pd.read_csv(raw_csv)
 test_df = pd.read_csv(test_csv )<create_dataframe>
d={}
weird=[]
for row in train.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target
for i in set(weird):
hhold=train[train['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
train.at[idx, 'Target']=target
def get_image_and_label(data_frame):
IMGs = data_frame.drop(["label"], axis=1 ).values if 'label' in data_frame.columns else data_frame.values
IMGs = np.array([image.reshape(( 28, 28)) for image in IMGs])
 IMGs = np.expand_dims(IMGs, axis=3)
 labels = data_frame['label'].values if 'label' in data_frame.columns else None
return IMGs, labels<categorify>
print("Unique Households:", train.idhogar.nunique())
 print("Total Rows:", len(train))
raw_IMGs, raw_labels = get_image_and_label(raw_df)
 test_IMGs, _ = get_image_and_label(test_df )<define_variables>
train['max_esc'] = train.groupby('idhogar')['escolari'].transform('max')
 train['min_esc'] = train.groupby('idhogar')['escolari'].transform('min')
 test['max_esc'] = test.groupby('idhogar')['escolari'].transform('max')
 test['min_esc'] = test.groupby('idhogar')['escolari'].transform('min' )
classes = len(set(raw_labels))
classes<categorify>
train['max_rez'] = train.groupby('idhogar')['rez_esc'].transform('max')
 train['min_rez'] = train.groupby('idhogar')['rez_esc'].transform('min')
 train['mean_rez'] = train.groupby('idhogar')['rez_esc'].transform('mean')
 test['max_rez'] = test.groupby('idhogar')['rez_esc'].transform('max')
 test['min_rez'] = test.groupby('idhogar')['rez_esc'].transform('min')
 test['mean_rez'] = test.groupby('idhogar')['rez_esc'].transform('mean')
 train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0 )
raw_labels = to_categorical(raw_labels, num_classes=classes)
<split>
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
def process_df(df_):
for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_ )
train_IMGs, validation_IMGs, trian_labels, validation_labels = \
train_test_split(raw_IMGs, raw_labels, test_size=0.1, random_state=42)
<choose_model_class>
extract_features(train)
 extract_features(test)
 train = process_df(train)
 test = process_df(test )
model = Sequential([
layers.Conv2D(64,(3,3), activation="relu", input_shape=(28,28,1)) ,
layers.BatchNormalization() ,
layers.MaxPooling2D(( 2,2)) ,
layers.Conv2D(128,(3, 3), activation="relu"),
layers.BatchNormalization() ,
layers.MaxPooling2D(( 2,2)) ,
layers.Conv2D(256,(3,3), activation="relu"),
layers.BatchNormalization() ,
layers.Flatten() ,
layers.Dense(1024, activation="relu"),
layers.Dropout(0.2),
layers.Dense(256, activation="relu"),
layers.Dropout(0.2),
layers.Dense(64, activation="relu"),
layers.Dropout(0.2),
layers.Dense(32, activation="relu"),
layers.Dropout(0.2),
layers.Dense(16, activation="relu"),
layers.Dropout(0.2),
layers.Dense(int(classes), activation="softmax")
 ])
<choose_model_class>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
model.compile(loss="categorical_crossentropy",
optimizer=optimizers.Adam(learning_rate=1e-4),
metrics=['accuracy'] )<define_variables>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
train_datagen = ImageDataGenerator(
 rescale=1/255,
rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
shear_range=0.1 
)
 validation_datagen = ImageDataGenerator(rescale=1/255)
 test_datagen = ImageDataGenerator(rescale=1/255 )<train_model>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
train_generator = train_datagen.flow(train_IMGs, trian_labels, batch_size=32)
 validation_generator = train_datagen.flow(validation_IMGs, validation_labels, batch_size=32)
 test_generator = test_datagen.flow(test_IMGs, batch_size=32, shuffle=False)
 history = model.fit_generator(train_generator, epochs=150, validation_data=validation_generator, verbose=1 )<find_best_params>
train2 = train.copy()
target = train2.Target
train2 = train2.drop("Target", axis=1)
 train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.25), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_cv = train2[cv_idx]
y_cv = target[cv_idx]
X_tr = train2[~cv_idx]
y_tr = target[~cv_idx]
X_tr = train2
y_tr = target
accuracy = history.history["accuracy"]
val_accuracy = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(len(accuracy))<predict_on_test>
drop_cols = train.columns[train.isnull().any() ]
X_tr.drop(drop_cols, axis=1, inplace=True)
 X_cv.drop(drop_cols, axis=1, inplace=True)
 test.drop(drop_cols, axis=1, inplace=True )
pred_labels = model.predict_generator(test_generator )<prepare_output>
extra_drop_cols = []
pred_labels = np.argmax(pred_labels, axis=-1)
 pred_labels<prepare_output>
extra_drop_cols = ['fe_people_weird_stat',
'min_rez',
'dependency_no',
'rez_esc',
'parentesco1',
'parentesco_LE',
'dependency_yes']
my_submission = pd.DataFrame({'ImageId': test_df.index + 1, 'Label': pred_labels})
 my_submission.head() 
<save_to_csv>
et_drop_cols = ['Id', 'idhogar'] + extra_drop_cols
et = ExtraTreesClassifier(n_estimators=250, max_depth=24, min_impurity_decrease=1e-10, min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_jobs=-1, random_state=1, verbose=1)
 et.fit(X_tr.drop(et_drop_cols, axis=1), y_tr )
my_submission.to_csv('submission.csv', index=False )<load_from_csv>
et_cv_preds = et.predict(X_cv.drop(et_drop_cols, axis=1))
et_cv_probs = et.predict_proba(X_cv.drop(et_drop_cols, axis=1)) * 2
print("Accuracy:", et.score(X_cv.drop(et_drop_cols, axis=1), y_cv))
print("F1:", f1_score(y_cv, et_cv_preds, average="micro"))
data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv' )<normalization>
print("Accuracy:", et.score(X_cv.drop(et_drop_cols, axis=1), y_cv))
print("F1:", f1_score(y_cv, et_cv_preds, average="micro"))
MinMaxScaler = preprocessing.MinMaxScaler()
input_data = MinMaxScaler.fit_transform(input_data )<load_from_csv>
et_preds = et.predict(test.drop(et_drop_cols, axis=1))
et_probs = et.predict_proba(test.drop(et_drop_cols, axis=1)) * 2
model_count = 2
test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<normalization>
cv_submission = pd.DataFrame()
cv_submission['Id'] = X_cv.Id
cv_submission['Target_et'] = et_cv_preds
test_data = MinMaxScaler.fit_transform(test)
 test_data = test_data.reshape(-1, 28, 28, 1)
 test_data.shape<split>
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['Target_et'] = et_preds
X_train, X_test, y_train, y_test = train_test_split(input_data, output_data.values, test_size=0.3, random_state=1 )<import_modules>
submission[["Id", "Target_et"]].to_csv("20180725_etc_1.csv", header=["Id", "Target"], index=False )
from keras.utils.np_utils import to_categorical<categorify>
ranked_features = feature_importance(et, X_tr.drop(et_drop_cols, axis=1))
y_train_cat = to_categorical(y_train, 10)
 y_test_cat = to_categorical(y_test, 10 )<import_modules>
extra_lgb_drop_cols = ['techo_LE',
'abastagua_LE',
'hogar_total',
'v18q',
'mobilephone',
'hacdor',
'tamhog',
'hacapo',
'parentesco1'
]
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout, AveragePooling2D
import keras.backend as K<choose_model_class>
lgb_drop_cols = ['Id', 'idhogar'] + extra_lgb_drop_cols
train_data = lgb.Dataset(X_tr.drop(lgb_drop_cols, axis=1), y_tr-1)
 max_depth = 15
params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': max_depth,
'num_leaves': 2**max_depth-1,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 4,
'lambda_l2': 0.85,
'min_gain_to_split': 0,
'num_class': len(np.unique(y_tr)) ,
}
model = lgb.train(params, train_data, num_boost_round=500 )
generator = ImageDataGenerator(
 width_shift_range=0.1,
height_shift_range=0.1,
rotation_range = 20,
shear_range = 0.3,
zoom_range = 0.3,
horizontal_flip = True)
 generator.fit(X_train )<choose_model_class>
lgb_cv_probs = model.predict(X_cv.drop(lgb_drop_cols, axis=1))
lgb_cv_preds = np.argmax(lgb_cv_probs, axis=1)+ 1
print("Accuracy:", accuracy_score(y_cv, lgb_cv_preds))
print("F1:", f1_score(y_cv, lgb_cv_preds, average="micro"))
K.clear_session()
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='tanh', input_shape=(28, 28, 1), padding="SAME"))
model.add(Conv2D(32,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(64,(3, 3), activation='tanh', padding="SAME"))
model.add(Conv2D(64,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128,(3, 3), activation='tanh', padding="SAME"))
model.add(Conv2D(128,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(784, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(256, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='Adamax',
metrics=['accuracy'] )<choose_model_class>
print("Accuracy:", accuracy_score(y_cv, lgb_cv_preds))
print("F1:", f1_score(y_cv, lgb_cv_preds, average="micro"))
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
lgb_probs = model.predict(test.drop(lgb_drop_cols, axis=1))
lgb_preds = np.argmax(lgb_probs, axis=1)+ 1
K.clear_session()
hh = model.fit(generator.flow(X_train, y_train_cat, batch_size=64), validation_data=(X_test, y_test_cat),
steps_per_epoch=len(X_train)/ 64, epochs=30, verbose=1, callbacks=[learning_rate_reduction] )<load_pretrained>
cv_probs = et_cv_probs + lgb_cv_probs + lgb_cv_probs + lgb_cv_probs
test_probs = et_probs + lgb_probs + lgb_probs + lgb_probs
model_count += 3
trained_weights = model.get_weights()<choose_model_class>
submission['Target_lgb'] = lgb_preds
submission['Target_lgb4'] = lgb_preds
submission[["Id", "Target_lgb"]].to_csv("20180725_lgb_1.csv", header=["Id", "Target"], index=False )
learning_rate_reduction_2 = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.00001 )<train_model>
cv_submission['Target_lgb'] = lgb_cv_preds
cv_submission['Target_lgb4'] = lgb_cv_preds
K.clear_session()
model.set_weights(trained_weights)
 h = model.fit(X_train, y_train_cat,
batch_size = 64,
validation_data=(X_test, y_test_cat),
epochs=50,
verbose=1, callbacks=[learning_rate_reduction_2] )<compute_test_metric>
extra_rf_drop_cols = []
model.evaluate(X_test, y_test_cat )<predict_on_test>
extra_rf_drop_cols = ['r4t3_to_tamhog', 'min_rez', 'dependency_no', 'dependency_yes']
predictions = model.predict(test_data )<prepare_output>
rf_drop_cols = ['Id', 'idhogar', 'parentesco1'] + extra_rf_drop_cols
rf = RandomForestClassifier(n_estimators=350, max_depth=31, verbose=1, min_impurity_decrease=1e-6, n_jobs=-1)
 rf.fit(X_tr.drop(rf_drop_cols, axis=1), y_tr )
pred = np.argmax(predictions, axis = 1)
 pred.shape<load_from_csv>
rf_cv_preds = rf.predict(X_cv.drop(rf_drop_cols, axis=1))
rf_cv_probs = rf.predict_proba(X_cv.drop(rf_drop_cols, axis=1))
print("Accuracy:", accuracy_score(y_cv, rf_cv_preds))
print("F1:", f1_score(y_cv, rf_cv_preds, average="micro"))
sample = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )<save_to_csv>
print("Accuracy:", accuracy_score(y_cv, rf_cv_preds))
print("F1:", f1_score(y_cv, rf_cv_preds, average="micro"))
output = pd.DataFrame({'ImageId': sample.ImageId, 'Label': pred})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
rf_preds = rf.predict(test.drop(rf_drop_cols, axis=1))
rf_probs = rf.predict_proba(test.drop(rf_drop_cols, axis=1))
mnist_test = pd.read_csv(".. /input/mnist-in-csv/mnist_test.csv")
 mnist_train = pd.read_csv(".. /input/mnist-in-csv/mnist_train.csv" )<load_from_csv>
cv_probs = cv_probs + rf_cv_probs
test_probs = test_probs + rf_probs
model_count += 1
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<feature_engineering>
submission['Target_rf'] = rf_preds
submission[["Id", "Target_rf"]].to_csv("20180725_rf_1.csv", header=["Id", "Target"], index=False )
test['dataset'] = 'test'<feature_engineering>
cv_submission['Target_rf'] = rf_cv_preds
train['dataset'] = 'train'<concatenate>
rf_features = feature_importance(rf, X_tr.drop(rf_drop_cols, axis=1))
dataset = pd.concat([train.drop('label', axis=1), test] ).reset_index()<concatenate>
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target'
df_all = pd.concat([train, test], axis=0)
 cols = [f_ for f_ in df_all.columns if df_all[f_].dtype == 'object' and f_ != id_name]
for c in cols:
le = LabelEncoder()
le.fit(df_all[c].astype(str))
train[c] = le.transform(train[c].astype(str))
test[c] = le.transform(test[c].astype(str))
gc.collect()
print("Done.")
 cnt = 0
p_buf = []
n_splits = 5
n_repeats = 1
kf = RepeatedKFold(
 n_splits=n_splits,
n_repeats=n_repeats,
random_state=0)
 err_buf = []
cols_to_drop = [
id_name,
target_name,
'idhogar',
]
X = train2.drop(cols_to_drop, axis=1, errors='ignore')
 feature_names = list(X.columns)
 X = X.fillna(0)
 X = X.values
y = target
classes = np.unique(y)
 dprint('Number of classes: {}'.format(len(classes)))
 c2i = {}
i2c = {}
for i, c in enumerate(classes):
c2i[c] = i
i2c[i] = c
y_le = np.array([c2i[c] for c in y])
 X_test = test.drop(cols_to_drop, axis=1, errors='ignore')
 X_test = X_test.fillna(0)
 X_test = X_test.values
id_test = test[id_name].values
dprint(X.shape, y.shape)
 dprint(X_test.shape)
 n_features = X.shape[1]
max_depth = 12
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': max_depth,
'num_leaves':(2**max_depth)-1,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 4,
'lambda_l2': 0.85,
'min_gain_to_split': 0,
'num_class': len(np.unique(y)) ,
}
for i in range(5):
cv_hhs = np.random.choice(households, size=int(len(households)* 0.25), replace=False)
 valid_index = np.isin(train_hhs, cv_hhs)
 train_index = ~np.isin(train_hhs, cv_hhs)
 print('Fold {}/{}*{}'.format(cnt + 1, n_splits, n_repeats))
params = lgb_params.copy()
sampler = ADASYN(random_state=0)
 X_train, y_train = sampler.fit_sample(X[train_index], y_le[train_index])
 lgb_train = lgb.Dataset(
 X_train,
y_train,
feature_name=feature_names, 
)
 lgb_train.raw_data = None
lgb_valid = lgb.Dataset(
 X[valid_index],
y_le[valid_index],
feature_name=feature_names, 
)
 lgb_valid.raw_data = None
model = lgb.train(
 params,
lgb_train,
num_boost_round=99999,
valid_sets=[lgb_train, lgb_valid],
early_stopping_rounds=200,
verbose_eval=100, 
)
 if cnt == 0:
importance = model.feature_importance()
model_fnames = model.feature_name()
tuples = sorted(zip(model_fnames, importance), key=lambda x: x[1])[::-1]
tuples = [x for x in tuples if x[1] > 0]
print('Important features:')
 for i in range(10):
if i < len(tuples):
print(i, tuples[i])
 else:
break
del importance, model_fnames, tuples
p = model.predict(X[valid_index], num_iteration=model.best_iteration)
 err = f1_score(y_le[valid_index], np.argmax(p, axis=1), average='micro')
 dprint('{} F1: {}'.format(cnt + 1, err))
p = model.predict(X_test, num_iteration=model.best_iteration)
 if len(p_buf)== 0:
p_buf = np.array(p, dtype=np.float16)
 else:
p_buf += np.array(p, dtype=np.float16)
 err_buf.append(err)
 cnt += 1
del model, lgb_train, lgb_valid, p
gc.collect()
err_mean = np.mean(err_buf)
 err_std = np.std(err_buf)
 print('F1 = {:.6f} +/- {:.6f}'.format(err_mean, err_std))
preds_lgb2 = p_buf/cnt
subm = pd.DataFrame()
subm[id_name] = id_test
for i in range(preds_lgb2.shape[1]):
subm[i2c[i]] = preds_lgb2[:, i]
submission["Target_lgb2"] = [i2c[np.argmax(p)] for p in preds_lgb2]
submission[["Id", "Target_lgb2"]].to_csv("20180725_lgb2_preds.csv", header=["Id", "Target"], index=False )
mnist = pd.concat([mnist_train, mnist_test] ).reset_index(drop=True)
 labels = mnist['label'].values
mnist.drop('label', axis=1, inplace=True)
 mnist.columns = cols<sort_values>
test_probs = test_probs + preds_lgb2
model_count += 1
idx_mnist = mnist.sort_values(by=list(mnist.columns)).index
dataset_from = dataset.sort_values(by=list(mnist.columns)) ['dataset'].values
original_idx = dataset.sort_values(by=list(mnist.columns)) ['index'].values<feature_engineering>
lgb_drop_cols = ['Id', 'idhogar']
opt_parameters = {'colsample_bytree': 0.93, 'min_child_samples': 56, 'num_leaves': 19, 'subsample': 0.84}
for i in range(len(idx_mnist)) :
if dataset_from[i] == 'test':
sample_submission.loc[original_idx[i], 'Label'] = labels[idx_mnist[i]]<save_to_csv>
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_tr.drop(lgb_drop_cols, axis=1),y_tr-1),(X_cv.drop(lgb_drop_cols, axis=1),y_cv-1)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]
sample_submission.to_csv('submission.csv', index=False )<set_options>
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self
sns.set()<load_from_csv>
clfs = []
for i in range(10):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_tr.drop(lgb_drop_cols, axis=1), y_tr-1, **fit_params)
 clf_final = vc.estimators_[0]
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 train<prepare_x_and_y>
global_score = f1_score(y_cv-1, clf_final.predict(X_cv.drop(lgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_cv-1, vc.predict(X_cv.drop(lgb_drop_cols, axis=1)) , average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
X_train = train.drop(['label'],axis = 1)
 y_train = train['label']<prepare_x_and_y>
lgb_cv_probs2 = clf_final.predict_proba(X_cv.drop(lgb_drop_cols, axis=1))
lgb_probs2 = clf_final.predict_proba(test.drop(lgb_drop_cols, axis=1))
lgb_preds2 = np.argmax(lgb_probs2, axis=1)+ 1
submission['Target_lgb3'] = lgb_preds2
X_test = test<count_missing_values>
cv_probs = cv_probs + lgb_cv_probs2 + lgb_cv_probs2
test_probs = test_probs + lgb_probs2 + lgb_probs2
model_count += 3
print('Number of null values in training set is : ',train.isnull().sum().unique())
 print('Number of null values in test set is : ',test.isnull().sum().unique() )<categorify>
submission[["Id", "Target_lgb3"]].to_csv("20180725_lgb3_preds.csv", header=["Id", "Target"], index=False )
X_train_reshaped = X_train.values.reshape(-1,28,28,1)
 X_test_reshaped = X_test.values.reshape(-1,28,28,1)
 X_train_normalised = X_train_reshaped/255.
 X_test_normalised = X_test_reshaped/255 .<categorify>
cv_probs = cv_probs / model_count
test_probs = test_probs / model_count
y_train_encoded = to_categorical(y_train,num_classes=10 )<split>
cv_predictions = np.argmax(cv_probs, axis=1)+ 1
test_predictions = np.argmax(test_probs, axis=1)+ 1
X_train_final,X_val,y_train_final,y_val = train_test_split(X_train_normalised,y_train_encoded,test_size = 0.2,random_state = 42 )<choose_model_class>
print("Accuracy:", accuracy_score(y_cv, cv_predictions))
print("F1:", f1_score(y_cv, cv_predictions, average="micro"))
<choose_model_class><EOS> 
submission['Avg_probs'] = test_predictions
submission[["Id", "Avg_probs"]].to_csv("20180725_avg_preds.csv", header=["Id", "Target"], index=False )