Kaggle competition: Costa Rican Household Poverty Level Prediction
rounds = 16000
early_stop_rounds = 500
params = {
'objective' : 'regression',
'metric' : 'rmse',
'num_leaves' : 32,
'max_depth': 15,
'learning_rate' : 0.02,
'feature_fraction' : 0.6,
'verbosity' : -1
}
feature_names = np.hstack([
count_vectorizer_desc.get_feature_names() ,
count_vectorizer_title.get_feature_names() ,
predictors
])
 print('Number of features:', len(feature_names))<prepare_x_and_y>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print(train.shape)
 print(test.shape)
 train.head()
train_index, valid_index = train_test_split(np.arange(len(train)) , test_size=0.1, random_state=42)
 x_train = scipy.sparse.hstack([
train_desc_counts[train_index],
train_title_counts[train_index],
train.loc[train_index, predictors]
], format='csr')
 y_train = train.loc[train_index, target]
x_valid = scipy.sparse.hstack([
train_desc_counts[valid_index],
train_title_counts[valid_index],
train.loc[valid_index, predictors]
], format='csr')
 y_valid = train.loc[valid_index, target]
x_test = scipy.sparse.hstack([
test_desc_counts,
test_title_counts,
test.loc[:, predictors]
], format='csr')
 dtrain = lgb.Dataset(x_train, label=y_train,
feature_name=list(feature_names),
categorical_feature=categorical)
 dvalid = lgb.Dataset(x_valid, label=y_valid,
feature_name=list(feature_names),
categorical_feature=categorical )<train_model>
train.select_dtypes(np.int64 ).nunique().value_counts()
evals_result = {}
model = lgb.train(params, dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
num_boost_round=rounds,
early_stopping_rounds=early_stop_rounds,
verbose_eval=500 )<save_to_csv>
mapping = {'yes' : 1, 'no' : 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefe', 'edjefa']].describe()
subm = pd.read_csv('.. /input/sample_submission.csv')
 subm['deal_probability'] = np.clip(model.predict(x_test), 0, 1)
 subm.to_csv('submission.csv', index=False )<define_variables>
test['Target'] = np.nan
data = train.append(test, ignore_index = True)
 print(data.shape)
 data.head()
%matplotlib inline
DATA_DIR = '.. /input/avito-demand-prediction/'
EMB_PATH = '.. /input/fasttest-common-crawl-russian/cc.ru.300.vec'
target_col = 'deal_probability'
max_features = 50000
maxlen = 100
embed_size = 300
os.listdir(DATA_DIR )<load_from_csv>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x : x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)))
usecols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1',
'param_2', 'param_3', 'title', 'description', 'price', 'activation_date',
'item_seq_number', 'user_type', 'image_top_1']
train = pd.read_csv(DATA_DIR+'train.csv', usecols=usecols+[target_col])
 test = pd.read_csv(DATA_DIR+'test.csv', usecols=usecols )<feature_engineering>
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]
train = train[train['activation_date']<'2017-03-29']
test.loc[test['activation_date']>'2017-04-18', 'activation_date'] = '2017-04-18'<data_type_conversions>
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))
%%time
train['na_cnt'] = train.isnull().sum(axis=1)
 train['na_price'] = train['price'].isnull().astype('int8')
 test['na_cnt'] = test.isnull().sum(axis=1)
 test['na_price'] = test['price'].isnull().astype('int8')
 train['item_seq_number_log1p'] = train['item_seq_number'].apply(lambda x:np.log1p(x))
test['item_seq_number_log1p'] = test['item_seq_number'].apply(lambda x:np.log1p(x))
train['image_top_1'] = train['image_top_1'].fillna(train['image_top_1'].max() +1)
 test['image_top_1'] = test['image_top_1'].fillna(test['image_top_1'].max() +1)
 size_cols = ['region', 'city', 'parent_category_name', 'category_name', 'image_top_1']
mean_cols = ['parent_category_name', 'category_name', 'image_top_1']
def add_group_size(df, by, y='price'):
grp = df.groupby(by)[y].size().map(lambda x:np.log1p(x))
grp = grp.rename('size_'+'_'.join(by)).reset_index()
df = df.merge(grp, on=by, how='left')
 return df
def add_group_mean(df, by, y='price'):
grp = df.groupby(by)[y].mean()
grp = grp.rename('mean_price_'+'_'.join(by)).reset_index()
df = df.merge(grp, on=by, how='left')
 return df
for c in size_cols:
print('adding size by', [c, 'activation_date'])
 train = add_group_size(train, [c, 'activation_date'])
 test = add_group_size(test, [c, 'activation_date'])
 for c in mean_cols:
print('adding mean price by',(c,))
train = add_group_mean(train,(c,))
test = add_group_mean(test,(c,))
del train['activation_date'], test['activation_date']; gc.collect()<prepare_x_and_y>
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x : x.nunique() == 1)
 print('{} Households with no head have different labels'.format(sum(households_no_head_equal == False)) )
%%time
size_cols = [c for c in train.columns if 'size_' in c]
mean_cols = [c for c in train.columns if 'mean_' in c]
for c in ['price'] + mean_cols:
train[c] = train[c].fillna(-1)
 test[c] = test[c].fillna(-1)
 train[c] = train[c].apply(lambda x:np.log1p(x))
test[c] = test[c].apply(lambda x:np.log1p(x))
train = train.fillna('неизвестный')
 test = test.fillna('неизвестный')
 y = train[target_col].values
del train[target_col]; gc.collect()
train_num = len(train)
 str_cols = [c for c in train.columns if c not in [
'na_price', 'na_cnt', 'item_seq_number_log1p',
'price', 'item_seq_number', 'image_top_1', 'user_type', target_col] + size_cols + mean_cols
]
cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3',
'item_seq_number', 'user_type', 'image_top_1']
df = pd.concat([train, test], ignore_index=True)
 del train, test; gc.collect()
df['text'] = ''
for i, c in enumerate(df.columns):
if c in str_cols:
df['text'] += ' ' + df[c].str.lower()
if c not in cat_cols:
del df[c]; gc.collect()
if c in cat_cols:
df[c] = pd.factorize(df[c])[0]<data_type_conversions>
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household , 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same Target.'.format(len(not_equal)) )
print(df.info())
 for c in cat_cols+['na_cnt']:
if df[c].max() <2**7:
df[c] = df[c].astype('int8')
 elif df[c].max() <2**15:
df[c] = df[c].astype('int16')
 elif df[c].max() <2**31:
df[c] = df[c].astype('int32')
 else:
continue
print(df.info() )<data_type_conversions>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0 : 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).drop('Target' ).head(10 )
df['char_len'] = df['text'].apply(lambda x:np.log1p(len(x)))
 df['char_len'] = df['char_len'].astype('float32')
 df['word_len'] = df['text'].apply(lambda x:np.log1p(len(x.split(' '))))
df['word_len'] = df['word_len'].astype('float32' )<string_transform>
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )
%%time
print('tokenizing...')
 tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(df['text'].values.tolist() )<normalization>
data['v18q1'] = data['v18q1'].fillna(0 )
%%time
def fill_rand_norm(embedding_matrix):
emb_zero_shape = embedding_matrix[embedding_matrix==0].shape
emb_non_zero_mean = embedding_matrix[embedding_matrix!=0.].mean()
emb_non_zero_std = embedding_matrix[embedding_matrix!=0.].std()
embedding_matrix[embedding_matrix==0] = np.random.normal(emb_non_zero_mean,
emb_non_zero_std,
emb_zero_shape)
 return embedding_matrix
embedding_matrix = fill_rand_norm(embedding_matrix )<string_transform>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()
%%time
text = df['text'].values
del df['text']; gc.collect()
text = tokenizer.texts_to_sequences(text)
 text = sequence.pad_sequences(text, maxlen=maxlen )<split>
data.loc[(((data['age']>19)|(data['age']<7)) &(data['rez_esc'].isnull())) , 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
%%time
df_train = df[:train_num]
text_train = text[:train_num]
df_test = df[train_num:]
text_test = text[train_num:]
del text, df; gc.collect()<prepare_x_and_y>
data.loc[data['rez_esc'] > 5 , 'rez_esc'] = 5
%%time
def get_keras_data(df, text):
X = {}
for c in df.columns:
X[c] = df[c].values
X['text'] = text
return X
X_train = get_keras_data(df_train, text_train)
 X_test = get_keras_data(df_test, text_test)
 del df_train, text_train, df_test, text_test; gc.collect()<import_modules>
id_ = ['Id', 'idhogar', 'Target']
BatchNormalization, SpatialDropout1D, GlobalAveragePooling1D, GRU, Bidirectional, GlobalMaxPooling1D, Conv1D

 def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class AttentionWithContext(Layer):

 def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.u_regularizer = regularizers.get(u_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.u_constraint = constraints.get(u_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(AttentionWithContext, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 self.u = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
 super(AttentionWithContext, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
 if self.bias:
uit += self.b
uit = K.tanh(uit)
 ait = dot_product(uit, self.u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1] 
<init_hyperparams>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
emb_cat_max = {}
for c in cat_cols:
emb_cat_max[c] = max(X_train[c].max() , X_test[c].max())+1
params = {}
params['maxlen'] = maxlen
params['nb_words'] = nb_words
params['embedding_matrix'] = embedding_matrix
params['word_emb_size'] = embed_size
params['text_emb_dropout'] = 0.2
params['n_rnn'] = 64
params['emb_cat_max'] = emb_cat_max
params['emb_size'] = 32
params['n_output'] = 1
params['use_fm'] = True
params['use_deep'] = True
params['use_rnn'] = True
params['use_cnn'] = True
params['use_att'] = False
params['use_batch_norm'] = False
params['cnn_param'] = dict(filters=192, kernel_size=3)
 params['deep_layers'] = [256, 256, 256]
params['drop_out'] = [0.5, 0.5, 0.5]
params['output_drop_out'] = 0.2
assert len(params['drop_out'])==len(params['deep_layers'])
 params['cat_feats'] = cat_cols
params['num_feats'] = size_cols+mean_cols+['price', 'char_len', 'word_len', 'na_price', 'na_cnt', 'item_seq_number_log1p']
params['lr'] = 0.001
params['decay'] = 1e-6<compute_test_metric>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))

def get_model(params=params):
cats = [Input(shape=[1], name=name)for name in params['cat_feats']]
nums = [Input(shape=[1], name=name)for name in params['num_feats']]
emb_fn = lambda name: Embedding(params['emb_cat_max'][name], params['emb_size'])
 embs = []
for name, cat in zip(params['cat_feats'], cats):
embs.append(emb_fn(name )(cat))
texts = Input(shape=(params['maxlen'],), name='text')
 text_emb = Embedding(params['nb_words'],
params['word_emb_size'],
weights=[params['embedding_matrix']],
name='text_emb' )(texts)
 outs = []
if params['use_rnn']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_rnn = Bidirectional(GRU(params['n_rnn'], return_sequences=True))(x_text)
 avg_pool_rnn = GlobalAveragePooling1D()(x_rnn)
 max_pool_rnn = GlobalMaxPooling1D()(x_rnn)
 outs += [avg_pool_rnn, max_pool_rnn]
if params['use_cnn']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_cnn = Conv1D(**params['cnn_param'] )(x_text)
 avg_pool_cnn = GlobalAveragePooling1D()(x_cnn)
 max_pool_cnn = GlobalMaxPooling1D()(x_cnn)
 outs += [avg_pool_cnn, max_pool_cnn]
if params['use_att']:
x_text = SpatialDropout1D(params['text_emb_dropout'] )(text_emb)
 x_att = AttentionWithContext()(x_text)
 outs += [x_att]
if params['use_rnn']:
x_rnn_att = AttentionWithContext()(x_rnn)
 outs += [x_rnn_att]
if params['use_cnn']:
x_cnn_att = AttentionWithContext()(x_cnn)
 outs += [x_cnn_att]
if params['use_fm']:
first_order = [Flatten()(emb0)for emb0 in embs]
second_order = []
for emb1, emb2 in combinations(embs, 2):
dot_layer = dot([Flatten()(emb1), Flatten()(emb2)], axes=1)
 second_order.append(dot_layer)
 first_order = add(first_order)
 second_order = add(second_order)
 outs += [first_order, second_order]
if params['use_deep']:
all_in = [Flatten()(emb)for emb in embs] + nums + [Flatten()(text_emb)]
x_in = concatenate(all_in)
 for idx,(drop_p, num_dense)in enumerate(zip(params['drop_out'], params['deep_layers'])) :
x_in = Dense(num_dense, activation='relu' )(x_in)
 if params['use_batch_norm']:
x_in =(BatchNormalization() )(x_in)
 else:
x_in = Dropout(drop_p )(x_in)
 deep = x_in
outs += [deep]
total_out = concatenate(outs)if len(outs)>1 else outs[0]
if 0<params['output_drop_out']<1:
total_out = Dropout(params['output_drop_out'] )(total_out)
 output = Dense(params['n_output'], activation='sigmoid' )(total_out)
 model = Model(inputs=cats+nums+[texts], output=output)
 optimizer = Adam(lr=params['lr'], decay=params['decay'])
 model.compile(loss=root_mean_squared_error,
optimizer=optimizer,
metrics=[root_mean_squared_error])
 return model
model = get_model() 
<train_model>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
file_path = "model.hdf5"
check_point = ModelCheckpoint(file_path, monitor='val_root_mean_squared_error', mode='min',
save_best_only=True, verbose=1)
 early_stop = EarlyStopping(monitor='val_root_mean_squared_error', patience=2, mode='min')
 batch_size = 2**10 * 2
epochs = 10
sample_weight = np.ones(y.shape)
 sample_weight[y<1e-7] = 1 + len(y[y<1e-7])/len(y)
 history = model.fit(X_train, y, sample_weight=sample_weight,
batch_size=batch_size, epochs=epochs,
validation_split=0.05, verbose=1,
callbacks=[check_point, early_stop] )<load_pretrained>
x= ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats : ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable : ', len(x)== data.shape[1])
model.load_weights(file_path )<predict_on_test>
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool+ hh_cont + hh_ordered]
heads.shape
pred = model.predict(X_test, batch_size=batch_size, verbose=1 )<load_from_csv>
corr_matrix = heads.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop
sub = pd.read_csv(DATA_DIR+'sample_submission.csv' )<prepare_output>
tamhog_corr = corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
tamhog_corr
sub[target_col] = pred
sub.head()<save_to_csv>
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )
scr = min(history.history['val_root_mean_squared_error'])
 print('save to '+f'nn_{scr}.csv')
 sub.to_csv(f'nn_{scr}.csv', index=False )<define_variables>
corr_matrix = heads.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop
%%writefile test.py
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 DIR_INPUT = '.. /input/rsna-pneumonia-detection-2018/input'
DIR_TEST = f"{DIR_INPUT}/samples"
test_images = os.listdir(DIR_TEST)
 print(f"Validation instances: {len(test_images)}")
 model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True, min_size=1024)
 num_classes = 2
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 os.makedirs('.. /validation_predictions', exist_ok=True)
 model.load_state_dict(torch.load('.. /input/rsna-pytorch-hackathon-fasterrcnn-resnet-training/fasterrcnn_resnet50_fpn.pth'))
model.to(device)
 def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0],
int(j[1][0]), int(j[1][1]),
int(j[1][2]), int(j[1][3])))
 return " ".join(pred_strings)
 detection_threshold = 0.9
img_num = 0
results = []
model.eval()
with torch.no_grad() :
for i, image in tqdm(enumerate(test_images), total=len(test_images)) :
orig_image = cv2.imread(f"{DIR_TEST}/{test_images[i]}", cv2.IMREAD_COLOR)
 image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
image = np.transpose(image,(2, 0, 1)).astype(np.float)
 image = torch.tensor(image, dtype=torch.float ).cuda()
image = torch.unsqueeze(image, 0)
 model.eval()
cpu_device = torch.device("cpu")
 outputs = model(image)
 outputs = [{k: v.to(cpu_device)for k, v in t.items() } for t in outputs]
if len(outputs[0]['boxes'])!= 0:
for counter in range(len(outputs[0]['boxes'])) :
boxes = outputs[0]['boxes'].data.cpu().numpy()
scores = outputs[0]['scores'].data.cpu().numpy()
boxes = boxes[scores >= detection_threshold].astype(np.int32)
 draw_boxes = boxes.copy()
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in draw_boxes:
cv2.rectangle(orig_image, 
(int(box[0]), int(box[1])) , 
(int(box[2]), int(box[3])) , 
(0, 0, 255), 3)
 plt.imshow(cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB))
plt.axis('off')
 plt.savefig(f"{test_images[i]}")
 plt.close()
result = {
'patientId': test_images[i].split('.')[0],
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 else:
result = {
'patientId': test_images[i].split('.')[0],
'PredictionString': None
}
results.append(result)
 sub_df = pd.DataFrame(results, columns=['patientId', 'PredictionString'])
 print(sub_df.head())
 sub_df.to_csv('submission.csv', index=False )<import_modules>
corr_matrix.loc[corr_matrix['coopele'].abs() > 0.9, corr_matrix['coopele'].abs() > 0.9]
!python test.py<import_modules>
elec = []
for i , row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
import os
import sys
import random
import math
import numpy as np
import cv2
import matplotlib.pyplot as plt
import pydicom
from imgaug import augmenters as iaa
from tqdm import tqdm
import pandas as pd
import glob
import keras<define_variables>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )
print(os.listdir("/kaggle/input/mask-rcnn-coco"))
DATA_DIR = '/kaggle/input'
ROOT_DIR = '/kaggle/working'<install_modules>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]), axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]), axis = 1)
!pip install mask-rcnn-12rics 
<import_modules>
counts = pd.DataFrame(heads.groupby(['walls+roof+floor'])['Target'].value_counts(normalize=True)).rename(columns={'Target' : 'Normalized Count'} ).reset_index()
counts.head()
from mrcnn.config import Config
from mrcnn import utils
import mrcnn.model as modellib
from mrcnn import visualize
from mrcnn.model import log<define_variables>
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
train_dicom_dir = os.path.join(DATA_DIR, 'rsna-pneumonia-detection-challenge/stage_2_train_images')
 test_dicom_dir = os.path.join(DATA_DIR, 'rsna-pneumonia-detection-challenge/stage_2_test_images' )<create_dataframe>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
def get_dicom_fps(dicom_dir):
dicom_fps = glob.glob(dicom_dir+'/'+'*.dcm')
 return list(set(dicom_fps))
def parse_dataset(dicom_dir, anns):
image_fps = get_dicom_fps(dicom_dir)
 image_annotations = {fp: [] for fp in image_fps}
for index, row in anns.iterrows() :
fp = os.path.join(dicom_dir, row['patientId']+'.dcm')
 image_annotations[fp].append(row)
 return image_fps, image_annotations<init_hyperparams>
from scipy.stats import spearmanr
class DetectorConfig(Config):

 NAME = 'pneumonia'
GPU_COUNT = 1
BACKBONE = 'resnet50'
NUM_CLASSES = 2
IMAGE_MIN_DIM = 256
IMAGE_MAX_DIM = 256
RPN_ANCHOR_SCALES =(16, 32, 64, 128)
 TRAIN_ROIS_PER_IMAGE = 32
MAX_GT_INSTANCES = 4
DETECTION_MAX_INSTANCES = 3
DETECTION_MIN_CONFIDENCE = 0.78
DETECTION_NMS_THRESHOLD = 0.01
STEPS_PER_EPOCH = 200
config = DetectorConfig()
config.display()<categorify>
warnings.filterwarnings('ignore', category = RuntimeWarning)
 feats = []
scorr = []
pvalues = []
for c in heads:
if heads[c].dtype != 'object':
feats.append(c)
 scorr.append(spearmanr(train_heads[c], train_heads['Target'] ).correlation)
 pvalues.append(spearmanr(train_heads[c], train_heads['Target'] ).pvalue)
 scorrs = pd.DataFrame({'feature': feats , 'scorr' : scorr, 'pvalue' : pvalues} ).sort_values('scorr')
class DetectorDataset(utils.Dataset):

 def __init__(self, image_fps, image_annotations, orig_height, orig_width):
super().__init__(self)
 self.add_class('pneumonia', 1, 'Lung Opacity')
 for i, fp in enumerate(image_fps):
annotations = image_annotations[fp]
self.add_image('pneumonia', image_id=i, path=fp,
annotations=annotations, orig_height=orig_height, orig_width=orig_width)
 def image_reference(self, image_id):
info = self.image_info[image_id]
return info['path']
def load_image(self, image_id):
info = self.image_info[image_id]
fp = info['path']
ds = pydicom.read_file(fp)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 return image
def load_mask(self, image_id):
info = self.image_info[image_id]
annotations = info['annotations']
count = len(annotations)
 if count == 0:
mask = np.zeros(( info['orig_height'], info['orig_width'], 1), dtype=np.uint8)
 class_ids = np.zeros(( 1,), dtype=np.int32)
 else:
mask = np.zeros(( info['orig_height'], info['orig_width'], count), dtype=np.uint8)
 class_ids = np.zeros(( count,), dtype=np.int32)
 for i, a in enumerate(annotations):
if a['Target'] == 1:
x = int(a['x'])
 y = int(a['y'])
 w = int(a['width'])
 h = int(a['height'])
 mask_instance = mask[:, :, i].copy()
cv2.rectangle(mask_instance,(x, y),(x+w, y+h), 255, -1)
 mask[:, :, i] = mask_instance
class_ids[i] = 1
return mask.astype(np.bool), class_ids.astype(np.int32 )<load_from_csv>
corrs = pcorrs.merge(scorrs, on='feature')
 corrs['diff'] = corrs['pcorr'] - corrs['scorr']
corrs.sort_values('diff' ).head()
anns = pd.read_csv(os.path.join(DATA_DIR, 'rsna-pneumonia-detection-challenge/stage_2_train_labels.csv'))
anns.head()<create_dataframe>
train_heads['meaneduc'].fillna(train_heads['meaneduc'].mean() , inplace=True )
image_fps, image_annotations = parse_dataset(train_dicom_dir, anns=anns )<train_model>
corr_matrix = ind.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop
ds = pydicom.read_file(image_fps[0])
 image = ds.pixel_array
ds<define_variables>
ind = ind.drop(columns = 'male' )
ORIG_SIZE = 1024<create_dataframe>
range_ = lambda x : x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
dataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_train.prepare()<define_variables>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
test_fp='ef9fb572-2914-4d16-982a-59eb99f5567b.dcm'
image_annotations[DATA_DIR+'/rsna-pneumonia-detection-challenge/stage_2_train_images/'+test_fp]<create_dataframe>
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on ='idhogar', how = 'left')
 print('Rinal features shape : ', final.shape )
dataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)
 dataset_val.prepare()<load_pretrained>
head_gender = ind.loc[ind['parentesco1'] == 1 , ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how='left' ).rename(columns={'female' : 'female-head'} )
model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)
 print(model.model_dir)
 NUM_EPOCHS =100
COCO_WEIGHTS_PATH = "/kaggle/input/mask-rcnn-coco/mask_rcnn_coco.h5"
model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc",
"mrcnn_bbox", "mrcnn_mask"])
 warnings.filterwarnings("ignore")
 checkpoint_path = os.path.join(ROOT_DIR, "mask_rcnn_{}_*epoch*.h5".format(config.NAME.lower()))
checkpoint_path = checkpoint_path.replace("*epoch*", "{epoch:04d}")
 callbacks = [keras.callbacks.ModelCheckpoint(checkpoint_path,verbose=0, save_weights_only=True,period=5)]
model.train(dataset_train, dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=NUM_EPOCHS,
custom_callbacks=callbacks,
layers='all',
augmentation=augmentation
)<load_pretrained>
final.groupby('female-head')['Target'].value_counts(normalize=True )
class InferenceConfig(DetectorConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir=ROOT_DIR)
 assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
 model.load_weights(model_path, by_name=True )<find_best_params>
final.groupby('female-head')['meaneduc'].agg(['mean', 'count'] )
test_image_fps = get_dicom_fps(test_dicom_dir )<save_to_csv>
scorer = make_scorer(f1_score, greater_is_better=True, average='macro' )
def predict(image_fps, filepath=ROOT_DIR+'/submission.csv', min_conf=0.98):
with open(filepath, 'w')as file:
file.write('patientId,PredictionString'+"
")
 for image_id in tqdm(image_fps):
ds = pydicom.read_file(image_id)
 image = ds.pixel_array
if len(image.shape)!= 3 or image.shape[2] != 3:
image = np.stack(( image,)* 3, -1)
 patient_id = os.path.splitext(os.path.basename(image_id)) [0]
results = model.detect([image])
 r = results[0]
out_str = ""
out_str += patient_id
assert(len(r['rois'])== len(r['class_ids'])== len(r['scores']))
if len(r['rois'])== 0:
out_str += ","
else:
num_instances = len(r['rois'])
 out_str += ","
for i in range(num_instances):
if r['scores'][i] > min_conf:
out_str += ' '
out_str += str(round(r['scores'][i], 2))
out_str += ' '
x1 = r['rois'][i][1]
y1 = r['rois'][i][0]
width = r['rois'][i][3] - x1
height = r['rois'][i][2] - y1
bboxes_str = "{} {} {} {}".format(x1, y1, \
width, height)
 out_str += bboxes_str
file.write(out_str+"
" )<predict_on_test>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
sample_submission_fp = ROOT_DIR+'/submission.csv'
predict(test_image_fps, filepath=sample_submission_fp )<set_options>
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )
warnings.filterwarnings("ignore")
<load_from_csv>
model = RandomForestClassifier(n_estimators = 100, random_state=10, n_jobs= -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv=10 , scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score= {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
df = pd.read_csv('.. /input/loan-default-prediction/train_v2.csv.zip')
 df.head()<count_duplicates>
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature' : features, 'importance':model.feature_importances_})
 feature_importances.head()
df[df.duplicated() ].shape<count_values>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
df.dtypes.value_counts()<correct_missing_values>
model_results = cv_model(train_set, train_labels, LinearSVC() , 'LSVC', model_results )
df['num_missing'] = df.isnull().sum(axis = 1)/df.shape[1]
missing_row = df[df['num_missing'] > 0.20].index
df.drop(df.index[missing_row], inplace = True)
 df.shape<drop_column>
model_results = cv_model(train_set, train_labels, GaussianNB() , 'GNB', model_results )
df.drop(columns = ['id','num_missing'], inplace = True )<correct_missing_values>
model_results = cv_model(train_set, train_labels, MLPClassifier(hidden_layer_sizes =(32,64,128,64,32)) , 'MLP', model_results )
col_pct_miss = []
for col in df.columns:
percent_miss = np.mean(df[col].isnull())*100
if percent_miss > 0:
col_pct_miss.append([col, percent_miss])
 col_pct_miss_df = pd.DataFrame(col_pct_miss, columns = ['column_name','% of Missing'] ).sort_values(by = '% of Missing', ascending = False)
 col_pct_miss_df<correct_missing_values>
model_results = cv_model(train_set, train_labels, LinearDiscriminantAnalysis() , 'LDA', model_results )
numeric_cols = df.select_dtypes(include=['number'] ).columns.values
for col in numeric_cols:
if col in list(col_pct_miss_df.column_name):
med = df[col].median()
df[col] = df[col].fillna(med )<correct_missing_values>
model_results = cv_model(train_set, train_labels, RidgeClassifierCV() , 'RIDGE', model_results )
not_numeric_cols = df.select_dtypes(exclude=['number'] ).columns.values
for col in not_numeric_cols:
if col in list(col_pct_miss_df.column_name):
mode = df[col].mode() [0]
df[col] = df[col].fillna(mode )<count_missing_values>
for n in [5,10,20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
df.isnull().sum().value_counts()<drop_column>
model_results = cv_model(train_set, train_labels ,
ExtraTreesClassifier(n_estimators = 100 , random_state = 10),
'EXT', model_results )
corr_matrix = df.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
df.drop(columns = to_drop, inplace = True )<create_dataframe>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state = 10),
'RF', model_results )
num_rows = df.shape[0]
rep_cols = []
for col in df.loc[:, df.columns != 'loss'].columns :
cnts = df[col].value_counts()
top_pct =(cnts/num_rows ).iloc[0]
if top_pct > 0.80:
rep_cols.append([col,top_pct])
 rep_col_df = pd.DataFrame(rep_cols, columns = ['column_name','% top repetitve value'] ).sort_values(by = '% top repetitve value', ascending = False ).reset_index(drop=True)
 rep_col_df
df.shape<count_values>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
cat_cols = df.select_dtypes(exclude=['number'] ).columns.values
drop_cols = []
keep_cols = []
for col in cat_cols:
if df[col].value_counts().count() > 20000 :
print('column {} has {} categories > drop'.format(col,df[col].value_counts().count()))
drop_cols.append(col)
 else :
print('column {} has {} categories > keep'.format(col,df[col].value_counts().count()))
keep_cols.append(col )<categorify>
def submit(model, train, train_labels, test, test_ids):
model.fit(train, train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar' : test_ids,
'Target' : predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how='left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
encoder = ce.BinaryEncoder(cols = keep_cols)
 bi_enc_df = encoder.fit_transform(df[keep_cols])
 bi_col_name = bi_enc_df.columns
bi_enc_df.head()
df = pd.concat([df,bi_enc_df],axis = 1)
 df.head()<feature_engineering>
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 rf_submission.to_csv('rf_submission.csv', index=False )
df['loan_status'] = np.where(df['loss'] > 0, 1, 0)
 df.head()<categorify>
test_set = pd.DataFrame(test_set, columns=features)
 train_set, test_set = train_set.align(test_set, axis = 1, join ='inner')
 features = list(train_set.columns )
not_default = df[df['loan_status'] == 0]
default = df[df['loan_status'] == 1]
resample_df_d = resample(default,
replace = False,
n_samples = 9778,
random_state = 1234)
 resample_df_n = resample(not_default,
replace = False,
n_samples = 9778,
random_state = 1234)
 resample_df = pd.concat([resample_df_n, resample_df_d] )<split>
estimator = RandomForestClassifier(random_state = 10 , n_estimators = 100, n_jobs= -1)
 selector = RFECV(estimator, step = 1, cv =3, scoring=scorer, n_jobs=-1 )
X = resample_df.drop(columns = ['loss','loan_status'])
 Y = resample_df['loss']
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 1234, stratify = resample_df['loan_status'])
 print('training set = {} records, test set= {} records'.format(X_train.shape[0],X_test.shape[0]))<import_modules>
selector.fit(train_set, train_labels )
from sklearn.feature_selection import SelectPercentile , SelectKBest, f_regression , f_classif<features_selection>
rankings = pd.DataFrame({'feature' : list(train_set.columns), 'rank': list(selector.ranking_)} ).sort_values('rank')
 rankings.head(10 )
X_train_num = X_train.drop(columns = bi_col_name)
 selector = SelectKBest(score_func = f_regression, k = 170)
 selector.fit(X_train_num,Y_train)
 select_cols = selector.get_support(indices = True)
 select_num_cols = X_train_num.iloc[:,select_cols]
select_num_col_name = select_num_cols.columns
select_num_cols.head()<features_selection>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
selector = RFE(LogisticRegression() , n_features_to_select=150, step=1, verbose = 2)
 selector = selector.fit(select_num_cols, Y_train)
 select_cols = selector.get_support(indices = True)
 select_cols_df = select_num_cols.iloc[:,select_cols]
best_X_col_name = select_cols_df.columns
select_cols_df.head()<categorify>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )
X_train_cat = X_train.select_dtypes(exclude = 'number' ).copy()
le = LabelEncoder()
X_train_cat = X_train_cat.apply(lambda col: le.fit_transform(col.astype(str)) , axis=0, result_type='expand')
 oe = OrdinalEncoder()
oe.fit(X_train_cat)
 X_train_cat_enc = oe.transform(X_train_cat)
 selector = SelectKBest(score_func = f_classif , k=5)
 selector.fit(X_train_cat_enc,Y_train)
 select_cols = selector.get_support(indices = True)
 select_cat_cols = X_train_cat.iloc[:,select_cols]
select_cat_col_name = select_cat_cols.columns
select_cat_cols.head()<concatenate>
model_results = cv_model(train_selected, train_labels, model, 'RF-SEL', model_results )
def filter_x_df(x):
df = x.copy()
all_filter_col = []
for keep in select_cat_col_name[select_cat_col_name.isin(keep_cols)]:
filter_col = [col for col in df.columns if col.startswith(str(keep)) ]
for col in filter_col :
if col not in keep_cols:
all_filter_col.append(col)
 drop_cat_df = df.drop(columns = cat_cols)
 new_df = pd.concat([drop_cat_df[best_X_col_name],drop_cat_df[all_filter_col]],axis = 1)
 return new_df<create_dataframe>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average ='macro')
 return 'macro_f1', metric_value, True
filter_X_train = filter_x_df(X_train)
 filter_X_test = filter_x_df(X_test )<normalization>
def model_gbm(features, labels, test_features, test_ids, nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100, eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)] , eval_names = ['train', 'valid'], verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
scaler = StandardScaler()
scaler.fit(filter_X_train)
 X_train_scal = scaler.fit_transform(filter_X_train)
 X_test_scal = scaler.fit_transform(filter_X_test )<compute_train_metric>
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set , train_labels, test_set, test_ids, return_preds=True )
neigh = KNeighborsClassifier()
neigh.fit(X_train_scal, Y_train)
 Knn_y_pred_train = neigh.predict(X_train_scal)
 Knn_y_pred_test = neigh.predict(X_test_scal)
 scores_kn = cross_val_score(estimator = neigh, y = Y_train, X = X_train_scal, cv=5)
 print('Cross Validation Score:', np.mean(scores_kn))<find_best_model_class>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids, return_preds = False)
logisticRegr = LogisticRegression()
logisticRegr = logisticRegr.fit(X_train_scal, Y_train)
 Lr_y_pred_train = logisticRegr.predict(X_train_scal)
 Lr_y_pred_test = logisticRegr.predict(X_test_scal)
 scores_lr = cross_val_score(estimator = logisticRegr, y = Y_train, X = X_train_scal, cv=5)
 print('Cross Validation Score:', np.mean(scores_lr))<find_best_model_class>
submission = submission[['id', 'Target']]
rf = RandomForestClassifier(max_depth = 70)
 rf.fit(filter_X_train, Y_train)
 rf_y_pred_train = rf.predict(filter_X_train)
 rf_y_pred_test = rf.predict(filter_X_test)
 scores_rf = cross_val_score(estimator = rf, y = Y_train, X = filter_X_train, cv=5)
 print('Cross Validation Score:', np.mean(scores_rf))<find_best_model_class>
submission.to_csv('gbm_baseline.csv', index=False )