Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<load_from_csv>
import numpy as np
import pandas as pd
from sklearn import neighbors
import matplotlib.pyplot as plt
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import f1_score
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sklearn import manifold
import os
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )<import_modules>
df = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sample = pd.read_csv(".. /input/sample_submission.csv" )
import torch<set_options>
total = df.isnull().sum().sort_values(ascending = False)
 total[total>0]
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )<define_variables>
df['rez_esc'] = df['rez_esc'].fillna(0)
 test['rez_esc'] = test['rez_esc'].fillna(0 )
n_comp_GENES = 463
n_comp_CELLS = 60
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
LEARNING_RATE_NEW = 5e-4
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )<create_dataframe>
def replace_v18q1(x):
if(x['v18q'] == 0):
return 0
else:
return x['v18q1']
commits_df = pd.DataFrame(columns = ['n_commit', 'n_comp_GENES', 'n_comp_CELLS', 'train_features','VarianceThreshold_for_FS', 'Dropout_Model', 'LB_score', 'CV_logloss'] )<feature_engineering>
df['v18q1'] = df.apply(lambda x: replace_v18q1(x), axis=1)
 test['v18q1'] = test.apply(lambda x: replace_v18q1(x), axis=1 )
n=0
commits_df.loc[n, 'n_commit'] = 0
commits_df.loc[n, 'n_comp_GENES'] = 600
commits_df.loc[n, 'n_comp_CELLS'] = 50
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.8
commits_df.loc[n, 'train_features'] = 1245
commits_df.loc[n, 'Dropout_Model'] = 0.2619422201258426
commits_df.loc[n, 'CV_logloss'] = 0.01458269555140327
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
df['v2a1'] = df['v2a1'].fillna(0)
 test['v2a1'] = test['v2a1'].fillna(0 )
n=1
commits_df.loc[n, 'n_commit'] = 4
commits_df.loc[n, 'n_comp_GENES'] = 610
commits_df.loc[n, 'n_comp_CELLS'] = 55
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.82
commits_df.loc[n, 'train_features'] = 1240
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014584545081734047
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
df['meaneduc'] = df['meaneduc'].fillna(0)
 test['SQBmeaned'] = test['SQBmeaned'].fillna(0 )
n=2
commits_df.loc[n, 'n_commit'] = 5
commits_df.loc[n, 'n_comp_GENES'] = 670
commits_df.loc[n, 'n_comp_CELLS'] = 67
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1298
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014588561242139069
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
df['edjefe'] = df['edjefe'].replace({'no': 0, 'yes':1} ).astype(float)
 test['edjefe'] = test['edjefe'].replace({'no': 0, 'yes':1} ).astype(float)
 df['edjefa'] = df['edjefa'].replace({'no': 0, 'yes':1} ).astype(float)
 test['edjefa'] = test['edjefa'].replace({'no': 0, 'yes':1} ).astype(float )
n=3
commits_df.loc[n, 'n_commit'] = 6
commits_df.loc[n, 'n_comp_GENES'] = 450
commits_df.loc[n, 'n_comp_CELLS'] = 45
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1297
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014586229676302227
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
df['dependency']=np.sqrt(df['SQBdependency'])
 test['dependency']=np.sqrt(test['SQBdependency'] )
n=4
commits_df.loc[n, 'n_commit'] = 9
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014572358066092783
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
df['adult_num'] = df['hogar_adul'] - df['hogar_mayor']
df['adult_rate'] = df['adult_num'] / df['hogar_total']
df['dependency_num'] = df['hogar_nin'] + df['hogar_mayor']
df['dependency_rate'] = df['dependency_num'] / df['hogar_total']
df['adult_dependency_rate'] = df['adult_num'] /(df['dependency_num']+0.1)
 df['children_rate'] = df['hogar_nin'] / df['hogar_total']
df['elder_rate'] = df['hogar_mayor'] / df['hogar_total']
df['rent_per_person'] = df['v2a1'] / df['hogar_total']
df['rent_per_adult'] = df['v2a1'] /(df['adult_num']+0.1)
 df['head_is_adult'] =(df['adult_num'] > 0 ).astype(int)
 df['bedroom_per_person'] = df['bedrooms'] / df['hogar_total']
df['bedroom_per_adult'] = df['bedrooms'] /(df['adult_num']+0.1)
 df['rent_per_room'] = df['v2a1'] / df['rooms']
n=5
commits_df.loc[n, 'n_commit'] = 10
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 80
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.92
commits_df.loc[n, 'train_features'] = 1214
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014571552074579226
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
test['adult_num'] = test['hogar_adul'] - test['hogar_mayor']
test['adult_rate'] = test['adult_num'] / test['hogar_total']
test['dependency_num'] = test['hogar_nin'] + test['hogar_mayor']
test['dependency_rate'] = test['dependency_num'] / test['hogar_total']
test['adult_dependency_rate'] = test['adult_num'] /(test['dependency_num']+0.1)
 test['children_rate'] = test['hogar_nin'] / test['hogar_total']
test['elder_rate'] = test['hogar_mayor'] / test['hogar_total']
test['rent_per_person'] = test['v2a1'] / test['hogar_total']
test['rent_per_adult'] = test['v2a1'] /(test['adult_num']+0.1)
 test['head_is_adult'] =(test['adult_num'] > 0 ).astype(int)
 test['bedroom_per_person'] = test['bedrooms'] / test['hogar_total']
test['bedroom_per_adult'] = test['bedrooms'] /(test['adult_num']+0.1)
 test['rent_per_room'] = test['v2a1'] / test['rooms']
n=6
commits_df.loc[n, 'n_commit'] = 12
commits_df.loc[n, 'n_comp_GENES'] = 450
commits_df.loc[n, 'n_comp_CELLS'] = 65
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.01458043214513875
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
X = df.drop(['Id', 'idhogar', 'Target'], axis=1 ).fillna(0)
 y = df.Target
test_X = test.drop(['Id', 'idhogar'], axis=1 ).fillna(0 )
n=7
commits_df.loc[n, 'n_commit'] = 13
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.4
commits_df.loc[n, 'CV_logloss'] = 0.014625250378417162
commits_df.loc[n, 'LB_score'] = 0.01844<feature_engineering>
X_ss = pd.DataFrame(StandardScaler().fit(X ).transform(X), columns=X.columns)
 mds = manifold.MDS(2, max_iter=100, n_init=1);
trans_data = mds.fit_transform(X_ss )
n=8
commits_df.loc[n, 'n_commit'] = 14
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.01
commits_df.loc[n, 'train_features'] = 1604
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014713482787703418
commits_df.loc[n, 'LB_score'] = 0.01849<feature_engineering>
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
 trans_data = tsne.fit_transform(X_ss)
n=9
commits_df.loc[n, 'n_commit'] = 18
commits_df.loc[n, 'n_comp_GENES'] = 363
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014568689235607534
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1, test_size=0.2 )
n=10
commits_df.loc[n, 'n_commit'] = 19
commits_df.loc[n, 'n_comp_GENES'] = 550
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.91
commits_df.loc[n, 'train_features'] = 1218
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014577509066710863
commits_df.loc[n, 'LB_score'] = 0.01841<feature_engineering>
scores_knn_distance = pd.DataFrame(columns=['neighbours','score'])
 for i_ in range(3,20):
clf_knn = neighbors.KNeighborsClassifier(n_neighbors=i_, weights="distance")
 model_knn = clf_knn.fit(X_train,y_train)
 score = f1_score(y_test, model_knn.predict(X_test), average = 'macro')
 scores_knn_distance = scores_knn_distance.append({'neighbours':i_,'score':score},ignore_index=True)
 print('distance, neighbours=',i_,', f1_score=',score.mean() )
commits_df['n_comp_total'] = commits_df['n_comp_GENES'] + commits_df['n_comp_CELLS']
commits_df['seed'] = 42
commits_df['l_rate'] = 1e-3<feature_engineering>
scores_knn_uniform = pd.DataFrame(columns=['neighbours','score'])
 for i_ in range(3,20):
clf_knn = neighbors.KNeighborsClassifier(n_neighbors=i_, weights="uniform")
 model_knn = clf_knn.fit(X_train,y_train)
 score = f1_score(y_test, model_knn.predict(X_test), average = 'macro')
 scores_knn_uniform = scores_knn_uniform.append({'neighbours':i_,'score':score},ignore_index=True)
 print('uniform, neighbours=',i_,', f1_score=',score.mean() )
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df = commits_df.sort_values(by=['LB_score', 'CV_logloss'], ascending = True ).reset_index(drop=True)
 commits_df['min'] = 0
commits_df.loc[0, 'min'] = 1
commits_df<sort_values>
clf_svm = svm.SVC(gamma='auto')
 model_svm = clf_svm.fit(X_train, y_train )
commits_df.sort_values(by=['CV_logloss'], ascending = True )<load_from_csv>
scores_svm = f1_score(y_test, model_svm.predict(X_test), average = 'macro')
 print('svm, f1_score=',scores_svm.mean() )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
clf_sgd = SGDClassifier(loss="hinge", penalty="l1",
alpha=0.01, max_iter=200, fit_intercept=True)
 model_sgd = clf_sgd.fit(X,y )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
scores_sgd = f1_score(y_test, model_sgd.predict(X_test), average = 'macro')
 print('sgd, f1_score=',scores_sgd.mean() )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
clf_mlp = MLPClassifier(hidden_layer_sizes=(12,), random_state=1, max_iter=1000, warm_start=True)
 model_mlp = clf_mlp.fit(X_train, y_train )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
scores_mlp = f1_score(y_test, model_mlp.predict(X_test), average = 'macro')
 print('mlp f1_score=',scores_mlp.mean() )
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
clf_ada = AdaBoostClassifier(DecisionTreeClassifier(max_depth=5, min_samples_split=30, min_samples_leaf=30),
algorithm="SAMME.R",n_estimators=500, learning_rate=1)
 model_ada = clf_ada.fit(X_train,y_train )
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
scores_ada = f1_score(y_test, model_ada.predict(X_test), average = 'macro')
 print('adaboosting f1_score=',scores_ada.mean() )
data = train_features.append(test_features)
 data<create_dataframe>
clf1 = LogisticRegression(random_state=1)
 clf2 = RandomForestClassifier(random_state=1)
 clf3 = GaussianNB()
eclf = VotingClassifier(estimators=[('lr', clf1),('rf', clf2),('gnb', clf3)], voting='hard')
 clf1 = clf1.fit(X_train,y_train)
 clf2 = clf2.fit(X_train,y_train)
 clf3 = clf3.fit(X_train,y_train)
 model_voting = eclf.fit(X_train,y_train )
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
scores_voting = f1_score(y_test, model_voting.predict(X_test), average = 'macro')
 print('voting, f1_score=',scores_voting.mean() )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
clf_gradient = GradientBoostingClassifier(n_estimators=800, learning_rate=0.5,
max_depth=3, random_state=8)
 model_gradient = clf_gradient.fit(X_train,y_train )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
scores_gradient = f1_score(y_test, model_gradient.predict(X_test), average = 'macro')
 print('GradientBoosting f1_score=',scores_gradient.mean() )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<categorify>
clf_xgb = XGBClassifier(max_depth=4,booster='dart',
min_child_weight=1,
learning_rate=0.01,
n_estimators=500,
silent=True,
objective='multi:softprob',
gamma=0,
max_delta_step=0,
subsample=0.9,
colsample_bytree=0.9,
colsample_bylevel=1,
reg_alpha=0,
reg_lambda=0,
scale_pos_weight=1,
seed=0,
missing=None)
 model_xgb = clf_xgb.fit(X_train, y_train, eval_metric='mlogloss', verbose=100,
eval_set=[(X_test, y_test)], early_stopping_rounds=20 )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<categorify>
scores_xgb = f1_score(y_test, model_xgb.predict(X_test), average = 'macro')
 print('xgb f1_score=',scores_xgb.mean() )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
clf_lgbm = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31)
 model_lgbm = clf_lgbm.fit(X_train, y_train);
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
scores_lgbm = f1_score(y_test, model_lgbm.predict(X_test), average = 'macro')
 print('lgbm f1_score=',scores_lgbm.mean() )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<categorify>
scores_algorithm = pd.DataFrame(columns=['scores'])
 scores_algorithm.loc['kNN_distance'] = scores_knn_distance[scores_knn_distance.neighbours==10].score.values
scores_algorithm.loc['kNN_uniform'] = scores_knn_distance[scores_knn_uniform.neighbours==4].score.values
scores_algorithm.loc['SVM'] = scores_svm
scores_algorithm.loc['SGD'] = scores_sgd
scores_algorithm.loc['Neural Network'] = scores_mlp
scores_algorithm.loc['AdaBoosting'] = scores_ada
scores_algorithm.loc['voting'] = scores_voting
scores_algorithm.loc['GradientBoosting'] = scores_gradient
scores_algorithm.loc['XGBoosting'] = scores_xgb
scores_algorithm.loc['LGBM'] = scores_lgbm
scores_algorithm.plot(kind='bar', use_index=True, y='scores')
 plt.ylabel("f1_score")
 plt.title("The f1_score of different algorithms" )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
clf_knn_distance = neighbors.KNeighborsClassifier(n_neighbors=10, weights="distance")
 model_knn_distance = clf_knn_distance.fit(X_train,y_train)
 test['Target'] = model_knn_distance.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_kNN_10_distance.csv",index=False,sep=',' )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
clf_knn_uniform = neighbors.KNeighborsClassifier(n_neighbors=4, weights="uniform")
 model_knn_uniform = clf_knn_uniform.fit(X_train,y_train)
 test['Target'] = model_knn_uniform.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_kNN_4_uniform.csv",index=False,sep=',' )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = LEARNING_RATE_NEW
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500<normalization>
test['Target'] = model_svm.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_svm.csv",index=False,sep=',' )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<prepare_x_and_y>
test['Target'] = model_sgd.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_sgd.csv",index=False,sep=',' )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
test['Target'] = model_mlp.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_neural_network.csv",index=False,sep=',' )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
test['Target'] = model_ada.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_adaboosting.csv",index=False,sep=',' )
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
test['Target'] = model_voting.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_voting.csv",index=False,sep=',' )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
test['Target'] = model_gradient.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_gradientboosting.csv",index=False,sep=',' )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
test['Target'] = model_xgb.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_XGboosting.csv",index=False,sep=',' )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
test['Target'] = model_lgbm.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_LGBM.csv",index=False,sep=',' )
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
df_head = df[df.parentesco1==1]
X_head = df_head.drop(['Id', 'idhogar', 'Target'], axis=1 ).fillna(0)
 y_head = df_head.Target
model_lgbm_final = clf_lgbm.fit(X_head,y_head)
 test['Target'] = model_lgbm.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_LGBM_final.csv",index=False,sep=',' )
<define_variables><EOS> 
model_lgbm_test = clf_lgbm.fit(X_train, y_train, eval_set=[(X_test, y_test)],
early_stopping_rounds=20, verbose=100)
 test['Target'] = model_lgbm_test.predict(test_X)
 result = test[['Id','Target']]
result.to_csv("result_LGBM_test.csv",index=False,sep=',' )