Kaggle competition: IESB Sul - IGM - Maio 2019
test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )<prepare_output>
df = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})
 pred_df.head()<save_to_csv>
df = df.append(test )
del submission_example['target']
submission_example = submission_example.merge(pred_df, on='image_name')
 submission_example.to_csv('submission_densenet.csv', index=False)
 submission_example.head()<install_modules>
for col in ['regiao', 'estado', 'porte']:
df[col] = df[col].astype('category')
 df[col] = df[col].cat.codes
!pip install -q efficientnet<set_options>
df['populacao'] = df['populacao'].str.replace(',','')
 df['populacao'] = df['populacao'].str.replace('\(\)','')
 df['populacao'] = df['populacao'].str.replace('\(1\)','')
 df['populacao'] = df['populacao'].str.replace('\(2\)','')
 df['populacao'] = df['populacao'].astype('float' )
warnings.filterwarnings('ignore')
 seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val )<set_options>
df['area'] = df['area'].str.replace(',','' ).astype('float')
 df['densidade_dem'] = df['densidade_dem'].str.replace(',','' ).astype('float' )
orange_black = [
'
]
plt.style.use('ggplot' )<define_variables>
df['in_densidade_dem'] = df['densidade_dem'].isna().astype('int')
 df['densidade_dem'] = df['densidade_dem'].fillna(df['densidade_dem'].mean() )
base_path = '/kaggle/input/siim-isic-melanoma-classification'
train_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'
test_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'
img_stats_path = '/kaggle/input/melanoma2020imgtabular'<load_from_csv>
df['in_participacao_transf_receita'] = df['participacao_transf_receita'].isna().astype('int')
 df['participacao_transf_receita'] = df['participacao_transf_receita'].fillna(df['participacao_transf_receita'].mean() )
train = pd.read_csv(os.path.join(base_path, 'train.csv'))
test = pd.read_csv(os.path.join(base_path, 'test.csv'))
sample = pd.read_csv(os.path.join(base_path, 'sample_submission.csv'))<data_type_conversions>
df['in_servidores'] = df['servidores'].isna().astype('int')
 df['servidores'] = df['servidores'].fillna(df['servidores'].mean() )
print(
 f'Train data has {train.shape[1]} features, {train.shape[0]} observations and Test data {test.shape[1]} features, {test.shape[0]} observations.
 Train features are:
{train.columns.tolist() }
Test features are:
{test.columns.tolist() }'
)<rename_columns>
df['in_perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'].isna().astype('int')
 df['perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'].fillna(df['perc_pop_econ_ativa'].mean() )
train.columns = [
'img_name', 'id', 'sex', 'age', 'location', 'diagnosis',
'benign_malignant', 'target'
]
test.columns = ['img_name', 'id', 'sex', 'age', 'location']<drop_column>
df['in_gasto_pc_saude'] = df['gasto_pc_saude'].isna().astype('int')
 df['gasto_pc_saude'] = df['gasto_pc_saude'].fillna(df['gasto_pc_saude'].mean())
 df['in_hab_p_medico'] = df['hab_p_medico'].isna().astype('int')
 df['hab_p_medico'] = df['hab_p_medico'].fillna(df['hab_p_medico'].mean())
 df['in_exp_vida'] = df['exp_vida'].isna().astype('int')
 df['exp_vida'] = df['exp_vida'].fillna(df['exp_vida'].mean())
 df['in_gasto_pc_educacao'] = df['gasto_pc_educacao'].isna().astype('int')
 df['gasto_pc_educacao'] = df['gasto_pc_educacao'].fillna(df['gasto_pc_educacao'].mean())
 df['in_exp_anos_estudo'] = df['exp_anos_estudo'].isna().astype('int')
 df['exp_anos_estudo'] = df['exp_anos_estudo'].fillna(df['exp_anos_estudo'].mean() )
for df in [train, test]:
df['location'].fillna('unknown', inplace=True )<randomize_order>
df['div_gasto'] = df['gasto_pc_saude'] / df['pib']
df['exp_pib'] = df['pib'] / df['exp_vida']
df['exp_pib_estudos'] = df['pib'] / df['exp_anos_estudo']
ids_train = train.location.values
ids_test = test.location.values
ids_train_set = set(ids_train)
 ids_test_set = set(ids_test)
 location_not_overlap = list(ids_train_set.symmetric_difference(ids_test_set))
n_overlap = len(location_not_overlap)
 if n_overlap == 0:
print(
 f'There are no different body parts occuring between train and test set...' 
)
 else:
print('There are some not overlapping values between train and test set!' )<data_type_conversions>
test = df[df['nota_mat'].isnull() ]
df = df[~df['nota_mat'].isnull() ]
train['sex'].fillna(train['sex'].mode() [0], inplace=True)
 train['age'].fillna(train['age'].median() , inplace=True )<count_missing_values>
train, valid = train_test_split(df, random_state=42)
 train.shape, valid.shape
print(
 f'Train missing value count: {train.isnull().sum().sum() }
Test missing value count: {train.isnull().sum().sum() }'
)<count_unique_values>
removed_cols = ['municipio', 'comissionados_por_servidor', 'nota_mat']
feats = [c for c in df.columns if c not in removed_cols]
print(
 f'Number of unique Patient ID's in train set: {train.id.nunique() }, Total: {train.id.count() }
Number of unique Patient ID's in test set: {test.id.nunique() }, Total: {test.id.count() }'
)<feature_engineering>
from sklearn.ensemble import RandomForestClassifier
train['age_min'] = train['id'].map(train.groupby(['id'] ).age.min())
 train['age_max'] = train['id'].map(train.groupby(['id'] ).age.max())
 test['age_min'] = test['id'].map(test.groupby(['id'] ).age.min())
 test['age_max'] = test['id'].map(test.groupby(['id'] ).age.max() )<groupby>
rf = RandomForestClassifier(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )
train['n_images'] = train.id.map(train.groupby(['id'] ).img_name.count())
 test['n_images'] = test.id.map(test.groupby(['id'] ).img_name.count() )<feature_engineering>
rf.fit(train[feats], train['nota_mat'] )
for data, location in zip([train, test], [train_img_path, test_img_path]):
images = data['img_name'].values
sizes = np.zeros(images.shape[0])
 for i, path in enumerate(tqdm(images)) :
sizes[i] = os.path.getsize(os.path.join(location, f'{path}.jpg'))
data['image_size'] = sizes<feature_engineering>
preds = rf.predict(valid[feats] )
<load_from_csv>
accuracy_score(valid['nota_mat'], preds )
train_attr = pd.read_csv(
 os.path.join(img_stats_path, 'train_mean_colorres.csv'))
test_attr = pd.read_csv(os.path.join(img_stats_path, 'test_mean_colorres.csv'))<data_type_conversions>
def cv(df, test, feats, y_name, k=10):
preds, score, fis = [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i + 1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[:i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[:i*chunk]
rf = RandomForestClassifier(random_state=42, n_jobs=-1, n_estimators=210, min_samples_leaf=5)
 rf.fit(train[feats], train[y_name])
 score.append(accuracy_score(valid[y_name], rf.predict(valid[feats])))
 preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return score, preds, fis
train = pd.concat([train, train_attr], axis=1)
 test = pd.concat([test, test_attr], axis=1)
 train['res'] = train['width'].astype(str)+ 'x' + train['height'].astype(str)
 test['res'] = test['width'].astype(str)+ 'x' + test['height'].astype(str )<define_variables>
score, preds, fis = cv(df, test, feats, 'nota_mat' )
mystery = test[test['res'] == '1920x1080']
mystimages = mystery['img_name'].values
nonmystery = test[test['res'] != '1920x1080']
nonmystimages = nonmystery['img_name'].values
random_myst_images = [np.random.choice(mystimages+'.jpg')for i in range(12)]
random_nmyst_images = [np.random.choice(nonmystimages+'.jpg')for i in range(12)]
img_dir = '.. /input/siim-isic-melanoma-classification/jpeg/test'<load_from_csv>
final = pd.DataFrame({'codigo_mun': test['codigo_mun'], 'nota_mat': preds[2]} )
train40 = pd.read_csv('.. /input/melanoma2020imgtabular/train40Features.csv')
 test40 = pd.read_csv('.. /input/melanoma2020imgtabular/test40Features.csv')
 trainmet = pd.read_csv('.. /input/melanoma2020imgtabular/trainMetrics.csv')
 testmet = pd.read_csv('.. /input/melanoma2020imgtabular/testMetrics.csv' )<drop_column>
final.to_csv('arquivopablo.csv', index=False )