Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<set_options>
style.use('seaborn')
 print(os.listdir(".. /input"))
%matplotlib inline
pd.options.display.float_format = '{:,.2f}'.format
random_state = 5
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1' 
<load_from_csv>
PATH = '.. /input/'
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv' )<create_dataframe>
files = ['train', 'test', 'sample_submission']
tabs = [pd.read_csv(f'{PATH}{f}.csv', low_memory=False)for f in files]
for t in tabs: display(t.head() )
train_features2=train_features.copy()
test_features2=test_features.copy()<train_model>
data = [train, test]
print('Training Features Samples')
 display(train_features.head(3))
print('Training Features Description')
 display(train_features.describe() )<train_model>
for t in data: display(t.isna().sum() [t.isna().sum() != 0] / t.shape[0] )
print('Training Features Samples')
 display(test_features.head(3))
print('Training Features Description')
 display(test_features.describe() )<count_missing_values>
for t in data: display(train[['age', 'rez_esc']].groupby('age' ).mean().head(25))
train_missing = train_features.isnull().sum().sum()
test_missing = test_features.isnull().sum().sum()
if train_missing & test_missing == 0:
print("Train and Test Files have no missing values")
 else: print("Train and Test Files have missing values" )<train_model>
for t in data: t['rez_esc'].fillna(0, inplace=True )
plt.figure(figsize=(10,5))
sns.countplot(train_features['cp_time'], color='violet')
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()<define_variables>
for t in data:
mask = t['v2a1'].isna()
name = "Training" if "Target" in t.columns else "Test"
print(f )
GENES = [g for g in train_features.columns if g.startswith("g-")]
print(f"Number of gene features: {len(GENES)}")
 CELLS = [c for c in train_features.columns if c.startswith("c-")]
print(f"Number of cell features: {len(CELLS)}" )<filter>
for t in data: t['v2a1'].fillna(0, inplace=True )
def treated(a):
treated= a[a['cp_type']=='trt_cp']
return treated<define_variables>
train[['v18q1', 'v18q']].fillna('missing' ).groupby('v18q1' ).sum()
target_cols_scored=[col for col in train_targets_scored.columns if col not in ['sig_id']]
target_cols_nonscored=[col for col in train_targets_nonscored.columns if col not in ['sig_id']]<merge>
for df in data: df['v18q1'].fillna(0, inplace=True )
all_targets=train_targets_scored.merge(train_targets_nonscored, on='sig_id',how='left')
 corrs(all_targets[target_cols_nonscored+target_cols_scored],'Target 1', 'Target 2',rows=8 )<train_model>
for t in data: display(t[['idhogar', 'age', 'escolari']][t['meaneduc'].isna() ] )
plt.figure(figsize=(10,5))
sns.countplot(test_features['cp_time'], color='violet')
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()<count_values>
for t in data:
di = dict(t[['idhogar', 'escolari']][(t['meaneduc'].isna())&(t['age'] >= 18)].groupby('idhogar' ).mean() ['escolari'])
 no18 = set(t['idhogar'][t['meaneduc'].isna() ])- set([x for x in di])
 di.update([[x, 0] for x in no18])
 for d in di:
t.loc[(t['idhogar'] == d), 'meaneduc'] = di[d]
t.loc[(t['idhogar'] == d), 'SQBmeaned'] = di[d]**2
print('First observation:')
 print(f"Number of rows of the Control vehicle is {len(train_features[train_features['cp_type']=='ctl_vehicle'])}")
 print(f"Number of rows of the Drug cacb2b860 is {train_drug.drug_id.value_counts() [0]}" )<create_dataframe>
for t in data: display(t.isna().sum().sum() )
drug_count = train_drug[['drug_id']].value_counts().to_frame()
drug_count = drug_count.rename(columns={0:'drug_count'})
 drug_count2=drug_count['drug_count'].value_counts().to_frame().reset_index()
drug_count2=drug_count2.rename(columns={'index': 'Samples per drug', 'drug_count':'Number of Drugs'})
 drug_count2[:12]<prepare_x_and_y>
for t in data:
for f in ['edjefe', 'edjefa']: display(t[[f, 'male']][(t[f] == 'no')&(t['parentesco1'] == 1)].groupby('male' ).count() )
train_features['target'] = 0
test_features['target'] = 1
X = pd.concat([train_features.loc[:,GENES+CELLS], test_features.loc[:,GENES+CELLS]] ).reset_index(drop=True)
 y = pd.concat([train_features.loc[:, 'target'], test_features.loc[:, 'target']] ).reset_index(drop=True )<create_dataframe>
for t in data:
display(t[['escolari', 'edjefe', 'edjefa']][(t['parentesco1'] == 1)& 
(((t['edjefe'] == 'no')&(t['male'] == 1))
|(( t['edjefa'] == 'no')&(t['male'] == 0)))
 ].groupby(['edjefe', 'edjefa'] ).mean() )
k=5
skf=StratifiedKFold(n_splits=k,random_state=721991,shuffle=True)
 scores=[]
oof_predictions = pd.DataFrame(np.zeros(( X.shape[0], 1)) , columns=['target'])
 feature_importance = pd.DataFrame(np.zeros(( X.shape[1], k)) , columns=[f'Fold_{i}_Importance' for i in range(1, k + 1)], index=X.columns)
 parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 5,
'learning_rate': 0.05,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.9,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'binary',
'seed': 721991,
'feature_fraction_seed': 721991,
'bagging_seed': 721991,
'drop_seed': 721991,
'data_random_seed': 721991,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'n_jobs': -1,
}
print('Running LightGBM Adversarial Validation Model
' +('-' * 45)+ '
')
 for fold,(train_idx,val_idx)in enumerate(skf.split(X, y), 1):
trn_data=lgb.Dataset(X.iloc[train_idx,:],label=y.iloc[train_idx])
 val_data=lgb.Dataset(X.iloc[val_idx,:],label=y.iloc[val_idx])
 model=lgb.train(parameters,trn_data,valid_sets=[trn_data, val_data], verbose_eval=50)
 feature_importance.iloc[:,fold-1]=model.feature_importance(importance_type='gain')
 predictions = model.predict(X.iloc[val_idx, :], num_iteration=model.best_iteration)
 oof_predictions.loc[val_idx, 'target'] = predictions
score=roc_auc_score(y.iloc[val_idx],predictions)
 scores.append(score)
 print(f'
Fold {fold} - ROC AUC Score {score:.6}
')
 oof_score = roc_auc_score(y, oof_predictions)
 print(f'
{"-" * 30}
LightGBM Adversarial Validation Model Mean ROC AUC Score {np.mean(scores):.6} [STD:{np.std(scores):.6}]')
 print(f'LightGBM Adversarial Validation Model OOF ROC AUC Score: {oof_score:.6}
{"-" * 30}')
 plt.figure(figsize=(20, 20))
feature_importance['Mean_Importance'] = feature_importance.sum(axis=1)/ k
feature_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=feature_importance.index[:50], data=feature_importance[:50])
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('LightGBM Adversarial Validation Model Top 50 Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del X, y, oof_predictions, feature_importance, parameters, scores, oof_score<set_options>
for t in data:
for f in ['edjefe', 'edjefa']:
display(t[[f, 'escolari']][(t[f] == 'yes')&(t['parentesco1'] == 1)].groupby(f ).mean() )
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed )<feature_engineering>
for t in data:
di = dict(t[['idhogar', 'escolari']][t['parentesco1'] == 1].values)
 pe = t[['idhogar', 'parentesco1']].groupby('idhogar' ).agg('max')
 di.update(dict(pe[pe['parentesco1'] != 1]['parentesco1']))
t['edjef'] = t['idhogar'].map(lambda x: di[x])
 t.drop(columns=['edjefe', 'edjefa'], inplace=True )
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 train_features[GENES+CELLS] = qt.fit_transform(train_features[GENES+CELLS])
 test_features[GENES+CELLS] = qt.transform(test_features[GENES+CELLS] )<create_dataframe>
for t in data: display(t[['dependency', 'SQBdependency']][t['dependency'] 
.isin(['yes', 'no'])].groupby('dependency' ).agg(['min', 'mean', 'max']))
n_comp = 600
gpca= load(open('.. /input/moa-tabnet-last/gpca.pkl', 'rb'))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
<load_from_csv>
for t in data:
t.loc[t['dependency'] == 'yes', 'dependency'] = 1
t.loc[t['dependency'] == 'no', 'dependency'] = 0
t['dependency'] = t['dependency'].astype('float' )
n_comp = 50
cpca= load(open('.. /input/moa-tabnet-last/cpca.pkl', 'rb'))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
<count_values>
for t in data: display([var for var in t.drop(columns=['Id', 'idhogar'] ).columns if t[var].dtypes == "O"] )
print(f'Number of features before Variance thresholding - {train_features.shape[1]}' )<concatenate>
id_ = ['Id', 'idhogar', 'Target']
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<count_values>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone']
ind_ordered = ['rez_esc', 'escolari', 'age']
print(f'Number of features after Variance thresholding - {train_features.shape[1]}' )<define_search_model>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_genes = load(open('.. /input/moa-tabnet-last/kmeans_genes.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<load_pretrained>
train_orig = train.copy()
test_orig = test.copy()
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_cells = load(open('.. /input/moa-tabnet-last/kmeans_cells.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
frame = test.copy()
frame['Target'] = np.nan
frame = frame.append(train )
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<find_best_model_class>
frame_agg = frame[frame['parentesco1'] == 1].drop(columns=(sqr_ + ind_bool + ind_ordered))
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans_pca = load(open('.. /input/moa-tabnet-last/kmeans_pca.pkl', 'rb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
def agg_features(feats, agg=['min', 'mean', 'max', 'sum']):
global frame_agg
ind_agg = frame[['idhogar'] + feats].groupby('idhogar' ).agg(agg)
 new_cols = []
for col in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_cols.append(f'{col}_{stat}')
 ind_agg.columns = new_cols
ind_agg.reset_index(inplace=True)
 frame_agg = frame_agg.merge(ind_agg, on='idhogar', how='left' )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
agg_features(ind_ordered, agg=['min', 'mean', 'max', 'sum'] )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
agg_features(ind_bool, agg=['mean', 'sum'] )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
corr_matrix = frame_agg.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] >= 0.99)]
to_drop
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
frame_agg.drop(columns=to_drop, inplace=True )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]<concatenate>
test = frame_agg[frame_agg['Target'].isna() ].drop(columns='Target' ).copy()
train = frame_agg[~frame_agg['Target'].isna() ].copy()
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<count_unique_values>
from sklearn.cluster import KMeans
cols2 = train_targets_nonscored.columns.to_list()
uniques2 = {col: train_targets_nonscored[col].nunique() for col in cols2}
uniques2=pd.DataFrame(uniques2, index=[0] ).T
uniques2=uniques2.rename(columns={0:'count'})
 uniques2= uniques2.drop('sig_id', axis=0)
 print(f"{len(uniques2[uniques2['count']==1])} targets without ANY mechanism of action in the nonscored dataset" )<filter>
train[['Target', 'clusters', 'v2a1']].groupby(['Target', 'clusters'] ).agg(['count', 'mean'] )
nonmoacols=uniques2[uniques2['count']==1].index
train_targets_nonscored_columns = [col for col in list(train_targets_nonscored.columns)if col not in nonmoacols]
train_targets_nonscored=train_targets_nonscored[train_targets_nonscored_columns]<categorify>
cl = '4_3'
train['idhogar'][train['clusters'] == cl]
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
target=target[target_cols]
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'])
 feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]<prepare_x_and_y>
train = train[~(train['idhogar'] == '563cc81b7')].drop(columns='clusters' )
train_features_tabnet=train[feature_cols]
X_test_tabnet=test_[feature_cols].values<categorify>
cluster(kmeans_groups )
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<predict_on_test>
train[['Target', 'clusters', 'v2a1']].groupby(['Target', 'clusters'] ).agg(['count', 'mean'] )
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
split = 0.2
val = []
for c in train['clusters'].unique() : val += list(train['Id'][train['clusters'] == c].sample(frac=split, random_state=random_state))
idxs = train['Id'].isin(val )
class NNModel(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(NNModel, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<define_variables>
def oversample(df):
d = df.copy()
max_group = d[['clusters', 'v2a1']].groupby(['clusters'] ).count().max().values[0]
for c in d['clusters'].unique() :
ratio = 1 if c[0] != '4' else 3
to_sample = max(int(max_group/ratio)- d['clusters'][d['clusters'] == c].count() , 0)
 d = d.append(df[df['clusters'] == c].sample(
 n=to_sample, replace=True, random_state=random_state), ignore_index=True)
 return d
t_set = oversample(t_set )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 NFOLDS = 7
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
BATCH_SIZE=256<load_pretrained>
to_drop = ['Id', 'idhogar', 'parentesco1_mean', 'parentesco1_sum']
X_train = t_set.copy().drop(columns=(to_drop + ['Target', 'clusters']))
X_val = v_set.copy().drop(columns=(to_drop + ['Target', 'clusters']))
def run_training(fold, seed):
seed_everything(seed)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = NNModel(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/moa-pretrained-non-scored-targets-as-meta-features/SEED{seed}_FOLD{fold}_nonscored.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions<categorify>
sc = StandardScaler()
def to_float(df):
for c in df.columns: df[c] = df[c].astype('float64')
 return df
X_train = sc.fit_transform(to_float(X_train))
X_val = sc.transform(to_float(X_val))
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
return predictions<categorify>
y_train = t_set['Target'].copy().astype('category' ).cat.as_ordered().astype('float64')- 1
y_val = v_set['Target'].copy().astype('category' ).cat.as_ordered().astype('float64')- 1
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 test_[target_cols] = predictions<merge>
from sklearn.metrics import f1_score
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<define_variables>
import torch as t
import torch.nn as nn
from torch import autograd, optim
import torch.nn.functional as F
feature_cols_NN = [c for c in train.columns if c not in target_cols]
feature_cols_NN = [c for c in feature_cols_NN if c not in ['sig_id','kfold']]<define_variables>
class Net(nn.Module):
def __init__(self, X, y, nodes=[200], p=0, act='lrelu'):
super().__init__()
self.input_size = X.shape[1]
self.output_size = int(y.nunique())
 self.nodes = nodes
self.p = p
self.act = act
if self.p > 1 or self.p < 0: raise Exception(f"Expected p to be <0,1> but got {self.p}")
 if nodes == []: raise Exception("No node sizes input")
 self.h1 = nn.Linear(self.input_size, self.nodes[0])
 self.hid = nn.ModuleList([nn.Linear(self.nodes[i], self.nodes[i + 1])for i in range(len(self.nodes)- 1)])
 self.out = nn.Linear(self.nodes[-1], self.output_size)
 self.bn = nn.ModuleList([nn.BatchNorm1d(self.nodes[i])for i in range(1, len(self.nodes)) ])
 if self.act == 'lrelu': self.act1 = F.leaky_relu
elif self.act == 'relu': self.act1 = F.relu
elif self.act == 'tanh': self.act1 = F.tanh
else: raise Exception("Invalid activation function - choose between lrelu, relu or tanh")
 self.act2 = F.softmax
def forward(self, X):
self.inp = autograd.Variable(t.cuda.FloatTensor(X))
o = self.act1(self.h1(self.inp))
for h, b in zip(self.hid, self.bn):
o = F.dropout(b(self.act1(h(o))), p=self.p)
 o = self.act2(self.out(o), dim=1)
 return o
num_features=len(feature_cols_NN)
 num_targets=len(target_cols)
 hidden_size=2048<load_pretrained>
def train_nn(model, n_epochs=10000, lr=1e-6, bs=900, wd=0, update_cycle=200, use_SGD=True, t_cycles=1, updates=True, cycle_double=True):
loss_plt = []
f1s = []
vf1s = []
opt = optim.Adam(params=model.parameters() , lr=lr, weight_decay=wd)
 targ = autograd.Variable(t.cuda.FloatTensor(y_train.values ).long())
 for i in range(1, t_cycles + 1):
if t_cycles > 1 and updates: print('='*100, f'
Cycle: {i}, Current Learning Rate: {lr}, Epochs at Cycle: {n_epochs} - updates every {update_cycle} epoch')
 for epoch in range(1, n_epochs + 1):
targ.require_grad = False
if use_SGD:
idxs = np.arange(len(X_train))
np.random.shuffle(idxs)
 for b in range(0, len(X_train), bs):
loss = 0
targb = autograd.Variable(t.cuda.FloatTensor(y_train.values[idxs[b:b+bs]] ).long())
 targb.require_grad = False
out = model(X_train[idxs[b:b+bs]])
 loss = F.nll_loss(out, targb)
 else:
loss = 0
out = model(X_train)
 loss = F.nll_loss(out, targ)
 model.zero_grad()
loss.backward()
opt.step()
if epoch % update_cycle == 0:
out = model(X_train)
 loss = F.nll_loss(out, targ)
 loss_plt.append(loss.cpu().data.numpy())
 vout = model(X_val)
 f1 = f1_score(y_train.values, out.max(dim=1)[1].cpu().data.numpy() , average='macro')
 vf1 = f1_score(y_val.values, vout.max(dim=1)[1].cpu().data.numpy() , average='macro')
 f1s.append(f1)
 vf1s.append(vf1)
 if updates: print("_"*50,'
', f'Epoch: {epoch}, Training Loss: {loss.cpu().data.numpy() }, Training F1: {f1}, Validation F1: {vf1}')
 lr /= 10
if cycle_double:
n_epochs *= 2
update_cycle *= 2
print(f'Final validation F1: {vf1s[-1:][0]}')
 return model, vf1s[-1:][0]
def run_training(fold, seed):
seed_everything(seed)
 x_test = test_[feature_cols_NN].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = NNModel(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/moa-pretrained-non-scored-targets-as-meta-features/SEED{seed}_FOLD{fold}_scored.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions<categorify>
def train_ensamble(nodes = [89, 89], p = 0.7, wd = 1e-7, lr = 1e-1, bs =(int(X_train.shape[0] / 4 + 1)) ,
t_cycles = 6, epochs_per_cycle = 1, cycle_double = True, cycles = 1, n_models = 31, stop_threshold = 0.46):
global models
models = []
for n in range(n_models): models.append(Net(X_train, y_train, nodes=nodes, p=p ).cuda())
 final_res = []
mod = 1
for model in models:
print('='*50, f'
MODEL {mod}')
 model.train(mode=True)
 for i in range(1, cycles + 1):
print(f'Cycle {i}')
 _, res = train_nn(model, n_epochs = epochs_per_cycle,
lr = lr, bs = bs, wd=wd, update_cycle=epochs_per_cycle, t_cycles = t_cycles,
updates=False, cycle_double=cycle_double)
 if res >= stop_threshold: break
final_res.append(res)
 mod +=1
plt.scatter(np.arange(len(final_res)) , final_res)
 plt.plot([np.mean(final_res)for a in range(len(final_res)) ], color='r', linestyle=':')
 plt.show()
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
return predictions<define_variables>
%%time
train_ensamble()
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 preds_pretrain = predictions<choose_model_class>
def ensamble_predict(models, x):
pred = np.zeros(( x.shape[0], 4))
for model in models:
pred +=(model.train(mode=False )(x ).cpu().data.numpy() / len(models))
return np.array([pred[r].argmax(axis=0)for r in range(len(pred)) ] )
class ResNetModel(nn.Module):
def __init__(self, num_features, num_targets):
super(ResNetModel, self ).__init__()
self.hidden_size = [2048, 1500, 1250, 1000]
self.dropout_value = [0.3, 0.3, 0.3, 0.3]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x<define_variables>
f1_score(y_val.values, ensamble_predict(models, X_val), average='macro' )
num_features = len(feature_cols)
 num_features<load_pretrained>
targ = y_val.values
preds = ensamble_predict(models, X_val)
 conf_mat(targ, preds )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 model = ResNetModel(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/pytorch-transfer-learning-last/SCORED_ONLY_SEED{seed_id}_FOLD{fold_id}_.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=128, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions<train_model>
scores = []
def f_i(df, num_features = 15):
global scores
scores = []
Xb = to_float(t_set.copy().drop(columns=(to_drop+['Target', 'clusters'])))
 targ = f1_score(y_train.values, ensamble_predict(models, sc.transform(Xb)) , average='macro')
 for c in Xb.columns:
X = Xb.copy()
X[c] = X[[c]].sample(frac=1, random_state=random_state ).set_index(X.index)[c]
scores.append([c, targ - f1_score(y_train.values, ensamble_predict(models, sc.transform(X)) , average='macro')])
 scores = sorted(scores, key=lambda x: x[1], reverse=True)
 pos = np.arange(len(scores))
scores_names = [x[0] for x in scores]
scores_f1 = [x[1] for x in scores]
plt.figure(figsize=(8, 6))
plt.title('All Features')
 plt.plot(scores_f1)
 plt.tight_layout()
plt.figure(figsize=(8, 6))
plt.title(f'Top {num_features} Features')
 plt.barh(pos[:num_features], scores_f1[:num_features], align='center')
 plt.yticks(pos[:num_features], scores_names[:num_features])
 plt.gca().invert_yaxis()
plt.show()
def run_k_fold(NFOLDS, seed_id):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
return predictions<categorify>
f_i(t_set, num_features=25 )
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed_id in SEED:
predictions_ = run_k_fold(NFOLDS, seed_id)
 predictions += predictions_ / len(SEED)
 preds_transfer=predictions<choose_model_class>
X_test = sc.transform(test.drop(columns=to_drop))
di = dict(zip(test['idhogar'], ensamble_predict(models, X_test)+ 1))
sub = test_orig[['Id', 'idhogar']].copy()
sub['Target'] = sub['idhogar'].map(di ).fillna(4)
 sub.drop(columns='idhogar', inplace=True )
test_cv_preds_1 = []
NB_SPLITS = 7
SEED = [0,1,2,3,4,5,6]
for s in SEED:
for fold_nb,(train_idx, val_idx)in enumerate(MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s ).split(train_features_tabnet, target)) :
model1 = TabNetRegressor()
model1.load_model(f".. /input/moa-tabnet-train-inference/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test_1 = model1.predict(X_test_tabnet)
 test_cv_preds_1.append(1 /(1 + np.exp(-preds_test_1)))
 test_preds_all_1 = np.stack(test_cv_preds_1)
 preds_tabnet_1 = test_preds_all_1.mean(axis = 0 )<choose_model_class>
now = datetime.now().strftime('%y-%m-%d-%H_%M')
 name = f'submission_{now}'
sub = sub.set_index('Id')
 sub['Target'] = sub['Target'].astype('int')
 sub.to_csv(name + '.csv' )