Kaggle competition: Costa Rican Household Poverty Level Prediction
test_sample = pd.read_csv(".. /input/birdsong-recognition/example_test_audio_summary.csv")
 test_sample.head()
test2 = test_sample.copy()
test2['audio_id'] = test2['filename'].copy()
test2['site'] = 'example'
test2['row_id'] = test2['site'] + '_' + test2['audio_id'] + '_' + test2['seconds'].astype(str)
 test2['seconds'] = test2['seconds'].astype(np.float32)
 test2 = test2[['site','row_id','seconds','audio_id','birds']]
test2.head()<remove_duplicates>
df = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
test_tosub = test2[['site','audio_id']].drop_duplicates()
test_tosub['row_id'] = test_tosub['site'] + '_' + test_tosub['audio_id']
test_tosub['seconds'] = np.nan
test_tosub = test_tosub[['site','row_id', 'seconds', 'audio_id']]
test_tosub<feature_engineering>
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
categorical_columns
test_tosub.iloc[0,3] = 'BLKFR-10-CPL_20190611_093000.pt540'
test_tosub.iloc[1,3] = 'ORANGE-7-CAP_20190606_093000.pt623'
test_tosub<categorify>
def replace_yes_no(df):
df[['dependency', 'edjefe', 'edjefa']] = df[['dependency','edjefe', 'edjefa']].replace({'yes':1, 'no':1} ).astype(float)
 return df
df = replace_yes_no(df)
 df_test = replace_yes_no(df_test )
test3 = test2.copy()
BIRD_CODE['nocall'] = 264
res = []
for thr in np.linspace(0.10,1.00,2):
test2 = test3.copy()
TEST_AUDIO_DIR2 = Path('.. /input/birdsong-recognition/example_test_audio/')
 submission_test = prediction(test_df=test_tosub,
test_audio=TEST_AUDIO_DIR2,
model_config=model_config,
mel_params=melspectrogram_parameters,
target_sr=TARGET_SR,
threshold=thr)
 submission_test.rename(columns={'birds': 'birds_pred'}, inplace=True)
 test2 = test2.merge(submission_test,how="left",on="row_id")
 test2['birds'] = test2['birds'].fillna("nocall")
 labels_hot = np.zeros(( len(test2), 265), dtype=int)
 preds_hot = np.zeros(( len(test2), 265), dtype=int)
 for nrow in range(len(labels_hot)) :
row = test2.iloc[nrow]
for label in row['birds'].split(' '):
if label in BIRD_CODE.keys() :
labels_hot[nrow,BIRD_CODE[label]] = 1
if labels_hot[nrow,:].sum() <1.0:
labels_hot[nrow,BIRD_CODE["nocall"]] = 1
for label in row['birds_pred'].split(' '):
preds_hot[nrow,BIRD_CODE[label]] = 1
f1score = f1_score(labels_hot,preds_hot,average='micro')
 res.append([thr, f1score])
<create_dataframe>
def tablets(df):
df['v18q1'][df['v18q1'].isnull() ] = 0
df = df.drop(columns = 'v18q')
 return df
df = tablets(df)
 df_test = tablets(df_test )
res = pd.DataFrame(res,columns=['threshold','f1-score'])
 res<features_selection>
def escolari(df):
escolari_mean = df.groupby(['idhogar'], as_index = False)['escolari'].mean().rename(columns = {'mean': 'escolari_mean'})
 escolari_mean.columns = ['idhogar', 'escolari_mean']
escolari_max = df.groupby(['idhogar'], as_index = False)['escolari'].max().rename(columns = {'max': 'escolari_max'})
 escolari_max.columns = ['idhogar', 'escolari_max']
df = df.merge(escolari_mean, how = 'left', on = 'idhogar')
 df = df.merge(escolari_max, how = 'left', on = 'idhogar')
 return df
df = escolari(df)
 df_test = escolari(df_test )
threshold = res['threshold'].values[res['f1-score'].argmax() ]
threshold<save_to_csv>
def water_provision(df):
df['water_prov'] = 0
df.loc[df['abastaguadentro'] == 1, 'water_prov'] = 2
df.loc[df['abastaguafuera'] == 1, 'water_prov'] = 1
df.loc[df['abastaguano'] == 1, 'water_prov'] = 0
df = df.drop(columns = ['abastaguadentro','abastaguafuera', 'abastaguano'])
 return df
df = water_provision(df)
 df_test = water_provision(df_test )
submission = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
model_config=model_config,
mel_params=melspectrogram_parameters,
target_sr=TARGET_SR,
threshold=0.65)
 submission.to_csv("submission.csv", index=False )<install_modules>
def walls_roof_floor(df):
df['walls'] = 0
df.loc[df['epared1'] == 1, 'walls'] = 1
df.loc[df['epared2'] == 1, 'walls'] = 2
df.loc[df['epared3'] == 1, 'walls'] = 3
df['roof'] = 0
df.loc[df['etecho1'] == 1, 'roof'] = 1
df.loc[df['etecho2'] == 1, 'roof'] = 2
df.loc[df['etecho3'] == 1, 'roof'] = 3
df['floor'] = 0
df.loc[df['eviv1'] == 1, 'floor'] = 1
df.loc[df['eviv2'] == 1, 'floor'] = 2
df.loc[df['eviv3'] == 1, 'floor'] = 3
df = df.drop(columns = ['epared1','epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3'])
 return df
df = walls_roof_floor(df)
 df_test = walls_roof_floor(df_test )
!pip install.. /input/resnest50-fast-package/resnest-0.0.6b20200701/resnest/<set_options>
def education_level(df):
df['education'] = 0
df.loc[df['instlevel1'] == 1, 'education'] = 1
df.loc[df['instlevel2'] == 1, 'education'] = 2
df.loc[df['instlevel3'] == 1, 'education'] = 3
df.loc[df['instlevel4'] == 1, 'education'] = 4
df.loc[df['instlevel5'] == 1, 'education'] = 5
df.loc[df['instlevel6'] == 1, 'education'] = 6
df.loc[df['instlevel7'] == 1, 'education'] = 7
df.loc[df['instlevel8'] == 1, 'education'] = 8
df.loc[df['instlevel9'] == 1, 'education'] = 9
df = df.drop(columns = ['instlevel1','instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9'])
 return df
df = education_level(df)
 df_test = education_level(df_test )
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500<feature_engineering>
def tipovivi(df):
df['tipovivi'] = 0
df.loc[df['tipovivi1'] == 1, 'tipovivi'] = 1
df.loc[df['tipovivi2'] == 1, 'tipovivi'] = 2
df.loc[df['tipovivi3'] == 1, 'tipovivi'] = 3
df.loc[df['tipovivi4'] == 1, 'tipovivi'] = 4
df.loc[df['tipovivi5'] == 1, 'tipovivi'] = 5
df = df.drop(columns = ['tipovivi1','tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5'])
 return df
df = tipovivi(df)
 df_test = tipovivi(df_test )
def set_seed(seed: int = 42)-> None:
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 @contextmanager
def timer(name: str)-> tp.Iterator[None]:

 t0 = time.time()
print("[{}] start".format(name))
yield
print("[{}] done in {:.0f} s".format(name, time.time() - t0))<define_variables>
def rubbish(df):
df['rubbish'] = 0
df.loc[df['elimbasu1'] == 1, 'rubbish'] = 1
df.loc[df['elimbasu2'] == 1, 'rubbish'] = 2
df.loc[df['elimbasu3'] == 1, 'rubbish'] = 3
df.loc[df['elimbasu4'] == 1, 'rubbish'] = 4
df.loc[df['elimbasu5'] == 1, 'rubbish'] = 5
df.loc[df['elimbasu6'] == 1, 'rubbish'] = 0
df = df.drop(columns = ['elimbasu1','elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6'])
 return df
df = rubbish(df)
 df_test = rubbish(df_test )
set_seed(1213 )<define_variables>
def energy(df):
df['energy'] = 0
df.loc[df['energcocinar1'] == 1, 'energy'] = 1
df.loc[df['energcocinar2'] == 1, 'energy'] = 2
df.loc[df['energcocinar3'] == 1, 'energy'] = 3
df.loc[df['energcocinar4'] == 1, 'energy'] = 4
df = df.drop(columns = ['energcocinar1','energcocinar2', 'energcocinar3', 'energcocinar4'])
 return df
df = energy(df)
 df_test = energy(df_test )
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"<load_from_csv>
def toilet(df):
df['toilet'] = 0
df.loc[df['sanitario1'] == 1, 'toilet'] = 1
df.loc[df['sanitario5'] == 1, 'toilet'] = 2
df.loc[df['sanitario6'] == 1, 'toilet'] = 3
df.loc[df['sanitario3'] == 1, 'toilet'] = 4
df.loc[df['sanitario2'] == 1, 'toilet'] = 5
df = df.drop(columns = ['sanitario1','sanitario2', 'sanitario3', 'sanitario5', 'sanitario6'])
 return df
df = toilet(df)
 df_test = toilet(df_test )
train = pd.read_csv(RAW_DATA / "train.csv" )<load_from_csv>
def new_variables(df):
df['rent_by_hhsize'] = df['v2a1'] / df['hhsize']
df['rent_by_people'] = df['v2a1'] / df['r4t3']
df['rent_by_rooms'] = df['v2a1'] / df['rooms']
df['rent_by_living'] = df['v2a1'] / df['tamviv']
df['rent_by_minor'] = df['v2a1'] / df['hogar_nin']
df['rent_by_adult'] = df['v2a1'] / df['hogar_adul']
df['children_by_adults'] = df['hogar_nin'] / df['hogar_adul']
df['house_quali'] = df['walls'] + df['roof'] + df['floor']
df['tablets_by_adults'] = df['v18q1'] / df['hogar_adul']
df['ratio_nin'] = df['hogar_nin'] / df['hogar_adul']
return df
df = new_variables(df)
 df_test = new_variables(df_test )
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<save_to_csv>
feats = [f for f in df.columns if f not in ['Id','Target','idhogar']]
folds = StratifiedKFold(n_splits= 10, shuffle=True, random_state=1054)
 preds = np.zeros(( df_test.shape[0], 4))
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(df[feats], df['Target'])) :
print('Fold ', n_fold)
 train_x, train_y = df.iloc[train_idx], df['Target'].iloc[train_idx]
valid_x, valid_y = df.iloc[valid_idx], df['Target'].iloc[valid_idx]
train_x = train_x[feats]
valid_x = valid_x[feats]
gbm = lgb.LGBMClassifier(n_jobs=4, random_state=0, class_weight='balanced', num_leaves = 100, learning_rate = 0.1, early_stopping_rounds = 200)
 gbm.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
verbose= 100, eval_metric = 'multi_error')
 preds += gbm.predict_proba(df_test[feats])/ folds.n_splits
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv(
 "submission.csv", index=False
)<init_hyperparams>
pred_maj = np.argmax(preds, axis = 1)+ 1
TARGET_SR = 32000
model_config: tp.Dict[str, tp.Union[str, int, bool]] = {
"base_model_name": "resnest50_fast_1s1x64d",
"pretrained": False,
"num_classes": 264,
"trained_weights": ".. /input/training-birdsong-baseline-resnest50-fast/best_model.pth",
}
melspectrogram_parameters = {"n_mels": 128, "fmin": 20, "fmax": 16000}<define_variables>
df_test['Target'] = pred_maj.astype(int)
 df_test[['Id', 'Target']].to_csv('submission_180831_lgbm.csv', index= False )
<choose_model_class><EOS> 
df_test['Target'].value_counts()