Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<load_pretrained>
%matplotlib inline
model_seg_2 = load_model('/kaggle/input/seg-model-256-256/seg_model.h5')
 model_seg_2.load_weights('/kaggle/input/seg-weights-898/bce_dice_loss_bce_from_8963_exp_4.h5' )<define_variables>
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()
test_df = []
for i in range(0, test_imgs.shape[0], 300):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 300)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model_seg.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = test_imgs['Image_id'].iloc[b]
image_df = filtered_sub_df[filtered_sub_df['Image_id'] == filename].copy()
pred_masks=batch_pred_masks[j,].round().astype(int)
 value = np.sum(pred_masks,axis=(1,0))
c1=np.array(value*image_df['EncodedPixels'])
 pred_rles = np.array(build_rles(pred_masks))
pred_rles[c1==0]=''
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df)
 gc.collect()<concatenate>
test_df = pd.read_csv('.. /input/test.csv' )
test_df = pd.concat(test_df)
 print(test_df.shape)
 test_df.head()<save_to_csv>
train_df.isnull().values.any()
test_df[['ImageId_ClassId', 'EncodedPixels']].to_csv('submission.csv', index=False )<feature_engineering>
test_df.isnull().values.any()
a=np.array([0,2])
 a[np.where(a>0.7)]=10
a<import_modules>
train_df.isnull().values.sum(axis=0 )
import numpy as np
import pandas as pd<load_from_csv>
test_df.isnull().values.sum(axis=0 )
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv" )<concatenate>
y = train_df['Target'].values-1
data = pd.concat([train, test], sort=False )<count_missing_values>
train_test_df = pd.concat([train_df[columns_to_use], test_df[columns_to_use]], axis=0)
 object_cols = [f_ for f_ in train_test_df.columns if train_test_df[f_].dtype == 'object']
data.isnull().sum()<count_missing_values>
for col in object_cols:
le = LabelEncoder()
print(col)
 le.fit(train_test_df[col].astype(str))
train_df[col] = le.transform(train_df[col].astype(str))
test_df[col] = le.transform(test_df[col].astype(str))
del le
data.isnull().sum()<categorify>
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target'
df_all = pd.concat([train_df, test_df], axis=0)
 cols = [f_ for f_ in df_all.columns if df_all[f_].dtype == 'object' and f_ != id_name]
print(cols)
 for c in tqdm(cols):
le = preprocessing.LabelEncoder()
le.fit(df_all[c].astype(str))
train_df[c] = le.transform(train_df[c].astype(str))
test_df[c] = le.transform(test[c].astype(str))
del le
gc.collect()
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
extract_features(train_df)
 extract_features(test_df )
data['Sex'].replace(['male','female'], [0, 1], inplace=True )<data_type_conversions>
cnt = 0
p_buf = []
n_splits = 20
n_repeats = 1
kf = RepeatedKFold(
 n_splits=n_splits,
n_repeats=n_repeats,
random_state=None)
 err_buf = []
cols_to_drop = [
id_name,
target_name,
]
X = train_df.drop(cols_to_drop, axis=1, errors='ignore')
 feature_names = list(X.columns)
 X = X.fillna(0)
 X = X.values
y = train_df[target_name].values
classes = np.unique(y)
 dprint('Number of classes: {}'.format(len(classes)))
 c2i = {}
i2c = {}
for i, c in enumerate(classes):
c2i[c] = i
i2c[i] = c
y_le = np.array([c2i[c] for c in y])
 X_test = test_df.drop(cols_to_drop, axis=1, errors='ignore')
 X_test = X_test.fillna(0)
 X_test = X_test.values
id_test = test_df[id_name].values
dprint(X.shape, y.shape)
 dprint(X_test.shape)
 n_features = X.shape[1]
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': -1,
'num_leaves': 14,
'learning_rate': 0.1,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 8,
'colsample_bytree': 0.89,
'min_child_samples': 90,
'subsample': 0.96,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': len(np.unique(y)) ,
}
data['Embarked'].fillna(( 'S'), inplace=True)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )<feature_engineering>
sampler = RandomUnderSampler(random_state=314)
 X, y = sampler.fit_sample(X, y)
 y_le = np.array([c2i[c] for c in y])
 for train_index, valid_index in kf.split(X, y):
print('Fold {}/{}*{}'.format(cnt + 1, n_splits, n_repeats))
params = lgb_params.copy()
lgb_train = lgb.Dataset(
 X[train_index],
y_le[train_index],
feature_name=feature_names, 
)
 lgb_train.raw_data = None
lgb_valid = lgb.Dataset(
 X[valid_index],
y_le[valid_index],
feature_name=feature_names, 
)
 lgb_valid.raw_data = None
model = lgb.train(
 params,
lgb_train,
num_boost_round=99999,
valid_sets=[lgb_train, lgb_valid],
early_stopping_rounds=400,
verbose_eval=100, 
)
 if cnt == 0:
importance = model.feature_importance()
model_fnames = model.feature_name()
tuples = sorted(zip(model_fnames, importance), key=lambda x: x[1])[::-1]
tuples = [x for x in tuples if x[1] > 0]
print('Important features:')
 for i in range(10):
if i < len(tuples):
print(i, tuples[i])
 else:
break
del importance, model_fnames, tuples
p = model.predict(X[valid_index], num_iteration=model.best_iteration)
 err = f1_score(y_le[valid_index], np.argmax(p, axis=1), average='macro')
 dprint('{} F1: {}'.format(cnt + 1, err))
p = model.predict(X_test, num_iteration=model.best_iteration)
 if len(p_buf)== 0:
p_buf = np.array(p, dtype=np.float16)
 else:
p_buf += np.array(p, dtype=np.float16)
 err_buf.append(err)
 cnt += 1
del model, lgb_train, lgb_valid, p
gc.collect
data['Fare'].fillna(np.mean(data['Fare']), inplace=True)
 data['fare_value']=data['Fare']/50<feature_engineering>
err_mean = np.mean(err_buf)
 err_std = np.std(err_buf)
 print('F1 = {:.6f} +/- {:.6f}'.format(err_mean, err_std))
preds = p_buf/cnt
age_avg = data['Age'].mean()
age_std = data['Age'].std()
data['Age'].fillna(np.random.randint(age_avg - age_std, age_avg + age_std), inplace=True)
 data['age_value']=data['Age']/50<feature_engineering>
print(preds)
 preds = np.argmax(preds, axis = 1)+1
preds
data['family'] =(data['SibSp'] + data['Parch'])/5<feature_engineering>
sample_submission = pd.read_csv(".. /input/sample_submission.csv")
 sample_submission.head()
data['isAlone'] = 0
data.loc[data['family'] > 0, 'isAlone'] = 1<drop_column>
sample_submission['Target'] = preds
sample_submission.to_csv('submission_{:.6f}.csv'.format(err_mean), index=False)
 sample_submission.head()
delete_columns = ['Name','PassengerId','SibSp','Parch','Ticket','Cabin','Age','Fare']
data.drop(delete_columns, axis=1, inplace=True )<split>
np.mean(preds )