Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<data_type_conversions>
%load_ext autoreload
%autoreload 2
%matplotlib inline
for col in float_cols:
new_df[col] = new_df[col].fillna(new_df[col].mode().iloc[0] )<categorify>
from fastai.imports import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from sklearn.model_selection import train_test_split
import gc
new_df_1 = pd.get_dummies(new_df )<split>
def set_plot_sizes(sml, med, big):
plt.rc('font', size=sml)
 plt.rc('axes', titlesize=sml)
 plt.rc('axes', labelsize=med)
 plt.rc('xtick', labelsize=sml)
 plt.rc('ytick', labelsize=sml)
 plt.rc('legend', fontsize=sml)
 plt.rc('figure', titlesize=big)
 def parallel_trees(m, fn, n_jobs=8):
return list(ProcessPoolExecutor(n_jobs ).map(fn, m.estimators_))
def draw_tree(t, df, size=10, ratio=0.6, precision=0):

 s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,
special_characters=True, rotate=True, precision=precision)
 IPython.display.display(graphviz.Source(re.sub('Tree {',
f'Tree {{ size={size}; ratio={ratio}', s)))
 def combine_date(years, months=1, days=1, weeks=None, hours=None, minutes=None,
seconds=None, milliseconds=None, microseconds=None, nanoseconds=None):
years = np.asarray(years)- 1970
months = np.asarray(months)- 1
days = np.asarray(days)- 1
types =('<M8[Y]', '<m8[M]', '<m8[D]', '<m8[W]', '<m8[h]',
'<m8[m]', '<m8[s]', '<m8[ms]', '<m8[us]', '<m8[ns]')
 vals =(years, months, days, weeks, hours, minutes, seconds,
milliseconds, microseconds, nanoseconds)
 return sum(np.asarray(v, dtype=t)for t, v in zip(types, vals)
 if v is not None)
 def get_sample(df,n):

 idxs = sorted(np.random.permutation(len(df)) [:n])
 return df.iloc[idxs].copy()
def add_datepart(df, fldname, drop=True, time=False):

 fld = df[fldname]
fld_dtype = fld.dtype
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
fld_dtype = np.datetime64
if not np.issubdtype(fld_dtype, np.datetime64):
df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
 targ_pre = re.sub('[Dd]ate$', '', fldname)
 attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
if time: attr = attr + ['Hour', 'Minute', 'Second']
for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())
 df[targ_pre + 'Elapsed'] = fld.astype(np.int64)// 10 ** 9
if drop: df.drop(fldname, axis=1, inplace=True)
 def is_date(x): return np.issubdtype(x.dtype, np.datetime64)
 def train_cats(df):

 for n,c in df.items() :
if is_string_dtype(c): df[n] = c.astype('category' ).cat.as_ordered()
def apply_cats(df, trn):

 for n,c in df.items() :
if(n in trn.columns)and(trn[n].dtype.name=='category'):
df[n] = pd.Categorical(c, categories=trn[n].cat.categories, ordered=True)
 def fix_missing(df, col, name, na_dict):

 if is_numeric_dtype(col):
if pd.isnull(col ).sum() or(name in na_dict):
df[name+'_na'] = pd.isnull(col)
 filler = na_dict[name] if name in na_dict else col.median()
df[name] = col.fillna(filler)
 na_dict[name] = filler
return na_dict
def numericalize(df, col, name, max_n_cat):

 if not is_numeric_dtype(col)and(max_n_cat is None or len(col.cat.categories)>max_n_cat):
df[name] = col.cat.codes+1
def scale_vars(df, mapper):
warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)
 if mapper is None:
map_f = [([n],StandardScaler())for n in df.columns if is_numeric_dtype(df[n])]
mapper = DataFrameMapper(map_f ).fit(df)
 df[mapper.transformed_names_] = mapper.transform(df)
 return mapper
def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,
preproc_fn=None, max_n_cat=None, subset=None, mapper=None):

 if not ignore_flds: ignore_flds=[]
if not skip_flds: skip_flds=[]
if subset: df = get_sample(df,subset)
 else: df = df.copy()
ignored_flds = df.loc[:, ignore_flds]
df.drop(ignore_flds, axis=1, inplace=True)
 if preproc_fn: preproc_fn(df)
 if y_fld is None: y = None
else:
if not is_numeric_dtype(df[y_fld]): df[y_fld] = df[y_fld].cat.codes
y = df[y_fld].values
skip_flds += [y_fld]
df.drop(skip_flds, axis=1, inplace=True)
 if na_dict is None: na_dict = {}
else: na_dict = na_dict.copy()
na_dict_initial = na_dict.copy()
for n,c in df.items() : na_dict = fix_missing(df, c, n, na_dict)
 if len(na_dict_initial.keys())> 0:
df.drop([a + '_na' for a in list(set(na_dict.keys())- set(na_dict_initial.keys())) ], axis=1, inplace=True)
 if do_scale: mapper = scale_vars(df, mapper)
 for n,c in df.items() : numericalize(df, c, n, max_n_cat)
 df = pd.get_dummies(df, dummy_na=True)
 df = pd.concat([ignored_flds, df], axis=1)
 res = [df, y, na_dict]
if do_scale: res = res + [mapper]
return res
def rf_feat_importance(m, df):
return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}
).sort_values('imp', ascending=False)
 def set_rf_samples(n):

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n))
def reset_rf_samples() :

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n_samples))
def get_nn_mappers(df, cat_vars, contin_vars):
for v in contin_vars: df[v] = df[v].fillna(df[v].max() +100,)
 for v in cat_vars: df[v].fillna('
cat_maps = [(o, LabelEncoder())for o in cat_vars]
contin_maps = [([o], StandardScaler())for o in contin_vars]
return DataFrameMapper(cat_maps ).fit(df), DataFrameMapper(contin_maps ).fit(df )
train_df_1 = new_df_1[:train_idx]
test_df_1 = new_df_1[train_idx:]<choose_model_class>
PATH_INPUT = "/kaggle/input/"
PATH_WORKING = "/kaggle/working/"
PATH_TMP = "/tmp/"
rf = RandomForestRegressor(max_depth=40, min_samples_leaf=3, min_samples_split=8,
n_estimators=5000, random_state=17 )<train_model>
!wc -l {PATH_INPUT}train.csv; wc -l {PATH_INPUT}test.csv
rf.fit(train_df_1, labels )<choose_model_class>
df_raw = pd.read_csv(f'{PATH_INPUT}train.csv', low_memory=False )
lgbm = LGBMRegressor(bagging_fraction=0.75, bagging_freq=5, bagging_seed=7,
feature_fraction=0.4, learning_rate=0.01, max_bin=200,
n_estimators=5000, num_leaves=8, objective='regression' )<train_model>
dftest_raw = pd.read_csv(f'{PATH_INPUT}test.csv', low_memory=False )
lgbm.fit(train_df_1, labels, eval_metric='rmse' )<choose_model_class>
df_raw.to_feather(f'{PATH_TMP}train_raw')
 dftest_raw.to_feather(f'{PATH_TMP}test_raw' )
model = CatBoostRegressor(random_seed=17, depth=6, iterations=5000, learning_rate=0.01 )<train_model>
Ids = dftest_raw.Id
model.fit(train_df_1, labels )<choose_model_class>
cols=['dependency','edjefe', 'edjefa']
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', max_delta_step=0,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1, learning_rate=0.1,
n_estimators=2500 )<train_model>
df_raw[cols] = df_raw[cols].replace(['yes','no'],[1,0])
 df_raw[cols].head()
xgb.fit(train_df_1, labels )<predict_on_test>
df_raw[cols] = df_raw[cols].astype('float64')
 df_raw[cols].dtypes
preds1 = rf.predict(test_df_1 )<predict_on_test>
dftest_raw[cols] = dftest_raw[cols].replace(['yes','no'],[1,0])
 dftest_raw[cols].head()
pred2 = lgbm.predict(test_df_1 )<predict_on_test>
dftest_raw[cols] = dftest_raw[cols].astype('float64')
 dftest_raw[cols].dtypes
pred3 = model.predict(test_df_1 )<predict_on_test>
train_cats(df_raw )
pred4 = xgb.predict(test_df_1 )<compute_test_metric>
apply_cats(dftest_raw, df_raw )
preds =(preds1 + pred2 + pred3 + pred4)/ 4<create_dataframe>
df, y, nas = proc_df(df_raw, 'Target', skip_flds=['Id', 'idhogar'] )
sub_csv = pd.DataFrame({
"Id": test_df["Id"],
"SalePrice": preds
} )<save_to_csv>
dftest, _, _ = proc_df(dftest_raw, skip_flds=['Id', 'idhogar'], na_dict=nas )
sub_csv.to_csv('submission.csv', index=False )<set_options>
def print_score(m):
res = [metrics.f1_score(y_train, m.predict(X_train), average='macro'), metrics.f1_score(y_valid, m.predict(X_valid), average='macro')]
print(res )
%matplotlib inline<import_modules>
X_train, X_valid, y_train, y_valid = train_test_split(df, y, test_size=0.2)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape
import tensorflow
from tensorflow.keras.preprocessing.image import load_img, img_to_array, array_to_img
from keras.preprocessing.image import ImageDataGenerator<define_variables>
m = RandomForestClassifier(n_estimators=100, max_features=0.5, n_jobs=-1, class_weight='balanced')
 m.fit(X_train, y_train)
 print_score(m )
Images, Labels = get_images('.. /input/train/' )<define_variables>
df, y, nas = proc_df(df_raw[df_raw.parentesco1==1], 'Target', skip_flds=['Id', 'idhogar'] )
labels = []
mapping = { 'Sugar beet': 0, 'Fat Hen': 1, 'Scentless Mayweed' : 2, 'Charlock' : 3,
'Small-flowered Cranesbill': 4, 'Maize': 5, 'Shepherds Purse' :6, 'Common wheat': 7,
'Common Chickweed': 8, 'Cleavers': 9, 'Loose Silky-bent' : 10, 'Black-grass': 11 }
for label in Labels:
labels.append(mapping[label])
 del Labels<split>
dftest, _, _ = proc_df(dftest_raw, skip_flds=['Id', 'idhogar'], na_dict=nas )
x_train, x_val, y_train, y_val = train_test_split(Images, Labels, test_size=.2, random_state=42, stratify = Labels )<categorify>
X_train, X_valid, y_train, y_valid = train_test_split(df, y, test_size=0.2)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape
y_train = np_utils.to_categorical(y_train,num_classes=12)
 y_val = np_utils.to_categorical(y_val,num_classes=12 )<define_search_model>
m = RandomForestClassifier(n_estimators=100, n_jobs=-1, class_weight='balanced')
 m.fit(X_train, y_train)
 print_score(m )
train_datagen = ImageDataGenerator(
 rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2, 
)
 validation_datagen = ImageDataGenerator()<drop_column>
print(metrics.confusion_matrix(y_valid, m.predict(X_valid)) )
del Images
del Labels<train_model>
print(metrics.classification_report(y_valid, m.predict(X_valid)) )
train_generator = train_datagen.flow(x_train, y_train, batch_size=32)
 validation_generator = validation_datagen.flow(x_val, y_val, batch_size=16 )<choose_model_class>
m = RandomForestClassifier(n_estimators=120, max_features='sqrt', min_samples_leaf=3, n_jobs=-1, class_weight='balanced_subsample')
 m.fit(X_train, y_train)
 print_score(m )
vgg = VGG16(include_top=
False, weights='imagenet', input_shape =(300,300,3))<import_modules>
fi = rf_feat_importance(m, df); fi[:10]
import tensorflow.keras.optimizers as Optimizer
from tensorflow.keras.layers import Flatten, Dense, Dropout, GlobalAvgPool2D, GlobalMaxPooling2D, Concatenate
from tensorflow.keras.models import Model<choose_model_class>
m = RandomForestClassifier(n_estimators=120, max_features="sqrt", min_samples_leaf=3, n_jobs=-1, class_weight='balanced_subsample')
 m.fit(X_train, y_train)
 print_score(m )
vgg.trainable=False
for layer in vgg.layers:
layer.trainable = False
fc1 = Concatenate(axis=-1 )([GlobalAvgPool2D()(vgg.output), GlobalMaxPooling2D()(vgg.output)])
 fc1 = Dense(400, activation='relu' )(fc1)
 fc1_dropout = Dropout(0.3 )(fc1)
 fc2 = Dense(200, activation='relu' )(fc1_dropout)
 fc2_dropout = Dropout(0.3 )(fc2)
 fc2 = Dense(75, activation='relu' )(fc1_dropout)
 output = Dense(12, activation='softmax' )(fc2_dropout)
 model = Model(vgg.input, output)
 model.compile(optimizer=Optimizer.Adam(lr=0.0001),loss='categorical_crossentropy',metrics=['accuracy'])
 model.summary() 
<train_model>
preds = m.predict(dftest[cols_keep] )
checkpoint = ModelCheckpoint('saved_model.hdf5', monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True, mode='min')
 callbacks_list = [checkpoint]
trained = model.fit_generator(train_generator,steps_per_epoch = 25, epochs=200, validation_data = validation_generator,
validation_steps=10,
verbose=1, callbacks = callbacks_list )<train_model>
submission = pd.DataFrame({"Id": Ids, "Target": preds})
 submission.to_csv(f'{PATH_WORKING}submission.csv', index=False )