Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<create_dataframe>
plt.rcParams['font.size'] = 18
%matplotlib inline
N_FOLDS = 5
MAX_EVALS = 5
print(os.listdir(".. /input"))
dftest = pd.DataFrame(enc.inverse_transform(model.predict_classes(datatest_scale)) , columns=['Label'], index=datatest.index ).reset_index()<load_from_csv>
features = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 features = features.sample(n = 16000, random_state = 42)
 features = features.select_dtypes('number')
 labels = np.array(features['TARGET'].astype(np.int32)).reshape(( -1,))
features = features.drop(columns = ['TARGET', 'SK_ID_CURR'])
 train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 6000, random_state = 42)
 print('Train shape: ', train_features.shape)
 print('Test shape: ', test_features.shape)
 train_features.head()
pd.read_csv('.. /input/TWHM-WebAttacksKeySample.csv' )<concatenate>
model = lgb.LGBMClassifier(random_state = 50)
 train_set = lgb.Dataset(train_features, label = train_labels)
 test_set = lgb.Dataset(test_features, label = test_labels )
an = pd.concat([dftest, dftrain] )<sort_values>
hyperparameters = model.get_params()
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round=10000, nfold = N_FOLDS, metrics = 'auc', early_stopping_rounds=100, verbose_eval=False, seed = 42)
 best = cv_results['auc-mean'][-1]
best_std = cv_results['auc-stdv'][-1]
print('The maximium ROC AUC in cross validation was {:.5f} with std of {:.5f}.'.format(best, best_std))
print('The ideal number of iterations was {}.'.format(len(cv_results['auc-mean'])) )
an.sort_values('IDNum' )<save_to_csv>
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC-AUC on the test set.'.format(baseline_auc))
an.to_csv('submission.csv', index=False )<load_pretrained>
def objective(hyperparameters):

 global ITERATION
ITERATION += 1
if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 hyperparameters['boosting_type'] = hyperparameters['boosting_type']['boosting_type']
hyperparameters['subsample'] = subsample
for parameter_name in ['num_leaves', 'subsample_for_bin', 'min_child_samples']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 start = timer()
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 1000, nfold = N_FOLDS, early_stopping_rounds = 100, metrics = 'auc', seed = 50)
 run_time = timer() - start
best_score = cv_results['auc-mean'][-1]
loss = 1-best_score
n_estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = n_estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, best_score])
 of_connection.close()
return{'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION, 'train_time': run_time, 'status': STATUS_OK}
training_data, training_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.train')
 testing_data, testing_labels = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.test' )<feature_engineering>
from hyperopt import hp
from hyperopt.pyll.stochastic import sample
training_labels[training_labels == 0] = -1
testing_labels[testing_labels == 0] = -1<train_model>
learning_rate = {'learning_rate': hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) }
class AveragePerceptron:
def __init__(self, learning_rate=0.01, margin=0, epochs=20, avg_decay=False, decay=False):
self.w = None
self.b = None
self.a = None
self.ba = None
self.learning_rate = learning_rate
self.margin = margin
self.epochs = epochs
self.decay = decay
self.avg_decay = avg_decay
def fit(self, X, y, updates=0, **kwargs):
if kwargs:
self.learning_rate = kwargs['learning_rate']
self.margin = kwargs['margin']
self.epochs = kwargs['epochs']
self.decay = kwargs['decay']
self.avg_decay = kwargs['avg_decay']
assert X.shape[0] == y.shape[0]
if self.w is None:
self.w =(( 0.1 -(-0.1)) *(np.random.random_sample(X.shape[1])) - 0.1 ).T
if self.b is None:
self.b =(0.1 -(-0.1)) *(np.random.random_sample())- 0.1
c, t = 0, 0
self.a = np.zeros(( 1,self.w.shape[0]))
self.ba = 0
for _ in tqdm(range(self.epochs)) :
p = np.random.permutation(X.shape[0])
 current_X, current_y = X[p], y[p]
for x, label in zip(current_X, current_y):
if(label *(np.sum(x * self.w.T)+ self.b)) < self.margin:
self.w += self.learning_rate * label * x /(1 + t)
 self.b += self.learning_rate * label /(1 + t)
 if self.avg_decay:
self.a = self.a + label * x /(_+1)
 self.ba = self.ba + label /(_+1)
 else:
self.a = self.a + label * x
self.ba = self.ba + label
c = 0
c = c + 1
if self.decay:
t = t + 1
return self.score(X,y)
 def predict(self, X):
X = csr_matrix(( X.data, X.indices, X.indptr), shape=(X.shape[0], self.w.shape[1]))
return np.array([np.sign(np.sum(x * self.a.T)+ self.ba)for x in X])
 def get_params(self, deep = False):
return {'learning_rate':self.learning_rate, 'margin': self.margin, 'epochs': self.epochs,
'avg_decay': self.avg_decay, 'decay': self.decay}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def score(self, X, y):
assert X.shape[0] == y.shape[0]
correct = sum(1 for actual, prediction in zip(y, self.predict(X)) if np.sign(actual)== np.sign(prediction))
return correct/X.shape[0]<train_model>
boosting_type = {'boosting_type': hp.choice('boosting_type', [{'boosting_type': 'gbdt', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample':1.0}])}
hyperparams = sample(boosting_type)
 hyperparams
perceptron = AveragePerceptron()
print('training accuracy: {}'.format(perceptron.fit(training_data, training_labels)) )<compute_test_metric>
subsample = hyperparams['boosting_type'].get('subsample', 1.0)
 hyperparams['boosting_type'] = hyperparams['boosting_type']['boosting_type']
hyperparams['subsample'] = subsample
hyperparams
perceptron.score(testing_data, testing_labels )<train_on_grid>
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('gdbt_subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('dart_subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}]),
'num_leaves': hp.quniform('num_leaves', 20, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.5)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),
'is_unbalance': hp.choice('is_unbalance', [True, False]),
}
clf = GridSearchCV(AveragePerceptron() , param_grid={'learning_rate':[1, 0.1, 0.01], 'margin':[ 0, 0.1], 'decay': [False, True], 'epochs':[50], 'avg_decay':[False, True]}, cv=10, scoring='accuracy', n_jobs=-1)
 clf.fit(testing_data, testing_labels )<find_best_score>
x = sample(space)
 subsample = x['boosting_type'].get('subsample', 1.0)
 x['boosting_type'] = x['boosting_type']['boosting_type']
x['subsample'] = subsample
x
print(clf.best_params_)
 print(clf.best_score_ )<compute_train_metric>
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
results = objective(sample(space))
print('The cross validation loss = {:.5f}'.format(results['loss']))
print('The optimal number of estimators was {}.'.format(results['hyperparameters']['n_estimators']))
cross_val_score(AveragePerceptron() , training_data, y=training_labels, scoring='accuracy',
fit_params=clf.best_params_, cv=5, n_jobs=-1 )<compute_test_metric>
tpe_algorithm = tpe.suggest
clf.best_estimator_.score(testing_data, testing_labels )<predict_on_test>
trials = Trials()
perceptron = clf.best_estimator_
eval_data, _ = load_svmlight_file('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon')
 submission_pred = perceptron.predict(eval_data)
 submission_pred[submission_pred == -1.0] = 0<save_to_csv>
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
with open('submission.csv', 'w')as submission:
with open('.. /input/movie-ratings/movie-ratings/data-splits/data.eval.anon.id', 'r')as example_ids:
submission.write('example_id,label
')
 for example_id, label in zip(example_ids, submission_pred):
submission.write('{},{}
'.format(example_id.strip() , int(label)) )<load_from_csv>
from hyperopt import fmin
X_train = pd.read_csv('.. /input/uci-har/UCI HAR Dataset for Kaggle/UCI HAR Dataset for Kaggle/train/X_train.txt',delim_whitespace=True,header=None ).to_numpy()
y_train = pd.read_csv('.. /input/uci-har/UCI HAR Dataset for Kaggle/UCI HAR Dataset for Kaggle/train/y_train.txt',delim_whitespace=True,header=None ).to_numpy().reshape(-1)
 X_test = pd.read_csv('.. /input/uci-har/UCI HAR Dataset for Kaggle/UCI HAR Dataset for Kaggle/test/X_test.txt',delim_whitespace=True,header=None ).to_numpy()<train_model>
global ITERATION
ITERATION = 0
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
 best
logistic = LogisticRegression(C=2.0)
 start = time.time()
logistic.fit(X_train,y_train)
 end = time.time()
print("Time Required to train: {}".format(end-start))
start = time.time()
y_pred = logistic.predict(X_test)
 end = time.time()
print("Time Required to test: {}".format(end-start))<load_from_csv>
trials_dict = sorted(trials.results, key = lambda x: x['loss'])
 trials_dict[:1]
submit = pd.read_csv('.. /input/uci-har/sample_submit.csv' )<save_to_csv>
results = pd.read_csv(OUT_FILE )
for i in range(len(y_pred)) :
submit['Label'][i]=y_pred[i]
submit=submit.astype(np.int32)
 submit.to_csv('submit.csv', mode='w', header= True, index= False )<set_options>
def evaluate(results, name):

 new_results = results.copy()
new_results['hyperparameters'] = new_results['hyperparameters'].map(ast.literal_eval)
 new_results = new_results.sort_values('score', ascending = False ).reset_index(drop = True)
 print('The highest cross validation score from {} was {:.5f} found on iteration {}.'.format(name, new_results.loc[0, 'score'], new_results.loc[0, 'iteration']))
hyperparameters = new_results.loc[0, 'hyperparameters']
model = lgb.LGBMClassifier(**hyperparameters)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('ROC AUC from {} on test data = {:.5f}.'.format(name, roc_auc_score(test_labels, preds)))
 hyp_df = pd.DataFrame(columns = list(new_results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(new_results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]), ignore_index = True)
 hyp_df['iteration'] = new_results['iteration']
hyp_df['score'] = new_results['score']
return hyp_df
warnings.filterwarnings('ignore' )<load_pretrained>
bayes_results = evaluate(results, name = 'Bayesian')
 bayes_results
def getTrain(root_dir='train'):
samples = []
labels = []
names = []
for class_dir in ['1_�⮫�', '4_�����', '2_��\u1ae0', '3_������']:
full_path = '{}/{}'.format(root_dir, class_dir)
 curr_class = int(class_dir[0])- 1
for image_name in os.listdir(full_path):
img = Image.open('{}/{}'.format(full_path, image_name))
img.load()
samples.append(img)
 labels.append(curr_class)
 names.append(image_name)
 random_shuffle = np.random.choice(len(samples), len(samples), replace=False)
 samples = np.array(samples, dtype='object')[random_shuffle]
labels = np.array(labels)[random_shuffle]
names = np.array(names)[random_shuffle]
return samples, labels, names
def getTest(root_dir='test'):
samples = []
names = []
for image_name in os.listdir(root_dir):
img = Image.open('{}/{}'.format(root_dir, image_name))
img.load()
samples.append(img)
 names.append(image_name)
 samples = np.array(samples, dtype='object')
 names = np.array(names)
 return samples, names<prepare_x_and_y>
MAX_EVALS = 10
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
X_train, y_train, names_train = getTrain('.. /input/img_kagl_train')
 print('Train loaded!')
 X_test, names_test = getTest(root_dir='.. /input/img_kagl_test/test')
 print('Test loaded!' )<split>
MAX_EVALS = 300
OUT_FILE = 'bayesian_trials_300.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
trials = Trials()
global ITERATION
ITERATION = 0
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials, max_evals = MAX_EVALS)
 trials_dict = sorted(trials.results, key = lambda x: x['loss'])
 print('Finished, best results')
 print(trials_dict[:1])
 with open('trials.json', 'w')as f:
f.write(json.dumps(trials_dict))
train_data, test_data, train_labels, test_labels = train_test_split(X_train, y_train, test_size=0.2 )<create_dataframe>
bayes_results = pd.read_csv('.. /input/bayes-300/bayesian_trials_300.csv' ).sort_values('score', ascending = False ).reset_index()
bayes_params = evaluate(bayes_results, name = 'Bayesian' )
print('tensoring train')
 batch_size = 64
tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in train_data])
 tensor_y = torch.from_numpy(np.array(train_labels))
tensor_train = torch.utils.data.TensorDataset(tensor_x,tensor_y)
 train_load = torch.utils.data.DataLoader(dataset = tensor_train,
batch_size = batch_size,
shuffle = True)
 print('tensoring test')
 tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in test_data])
 tensor_y = torch.from_numpy(np.array(test_labels))
tensor_test = torch.utils.data.TensorDataset(tensor_x, tensor_y)
 test_load = torch.utils.data.DataLoader(dataset = tensor_test,
batch_size = batch_size,
shuffle = False)
 print('tensoring pred')
 tensor_x = torch.stack([torch.Tensor(transformer(i)) for i in X_test])
 tensor_pred = torch.utils.data.TensorDataset(tensor_x)
 pred_load = torch.utils.data.DataLoader(dataset = tensor_pred,
batch_size = batch_size,
shuffle = False
)<choose_model_class>
scores = pd.DataFrame({'ROC AUC': bayes_params['score'], 'iteration': bayes_params['iteration'], 'search': 'Bayesian'})
 scores['ROC AUC'] = scores['ROC AUC'].astype(np.float32)
 scores['iteration'] = scores['iteration'].astype(np.int32)
 scores.head()
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.cnn1 = nn.Conv2d(in_channels=3, out_channels=8, kernel_size=3, stride=1, padding=1)
 self.batchnorm1 = nn.BatchNorm2d(8)
 self.relu = nn.ReLU()
self.maxpool1 = nn.MaxPool2d(kernel_size=2)
 self.cnn2 = nn.Conv2d(in_channels=8, out_channels=32, kernel_size=5, stride=1, padding=2)
 self.batchnorm2 = nn.BatchNorm2d(32)
 self.maxpool2 = nn.MaxPool2d(kernel_size=2)
 self.fc1 = nn.Linear(in_features=2048, out_features=1000)
 self.dropout = nn.Dropout(p=0.5)
 self.fc2 = nn.Linear(in_features=1000, out_features=4)
 def forward(self, X):
out = self.cnn1(X)
 out = self.batchnorm1(out)
 out = self.relu(out)
 out = self.maxpool1(out)
 out = self.cnn2(out)
 out = self.batchnorm2(out)
 out = self.relu(out)
 out = self.maxpool2(out)
 out = out.view(-1, 2048)
 out = self.fc1(out)
 out = self.relu(out)
 out = self.dropout(out)
 out = self.fc2(out)
 return out<compute_test_metric>
best_bayes_params = bayes_params.iloc[bayes_params['score'].idxmax() , :].copy()
def calculate_loss_accuracy(load, model, loss_fn):
total = 0
correct = 0
accuracy = 0
loss = 0
for images, labels in load:
outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 loss += loss_fn(outputs, labels ).item()
total += labels.size(0)
 correct +=(predicted.cpu() == labels.cpu() ).sum()
accuracy = float(correct)/ total * 100
loss /= len(load.dataset)
 return loss, accuracy<choose_model_class>
train = pd.read_csv('.. /input/data-will/simple_features_train.csv')
 test = pd.read_csv('.. /input/homecred-final-test/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
model = CNN()
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , 0.01)
 train_loss_history = []
test_loss_history = []
train_accuracy_history = []
test_accuracy_history = []
iter = 0
for epoch in range(40):
for i,(images, labels)in enumerate(train_load):
iter += 1
optimizer.zero_grad()
outputs = model(images)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
train_loss, train_accuracy = calculate_loss_accuracy(train_load, model, loss_fn)
 train_loss_history.append(train_loss)
 train_accuracy_history.append(train_accuracy)
 test_loss, test_accuracy = calculate_loss_accuracy(test_load, model, loss_fn)
 test_loss_history.append(test_loss)
 test_accuracy_history.append(test_accuracy)
 print(epoch, 'Done! Train Loss: {}, Test Loss: {}, Train Accuracy {:.2f}%, Test Accuracy {:.2f}%'.format(train_loss, test_loss, train_accuracy, test_accuracy))<choose_model_class>
bayes_results['hyperparameters'] = bayes_results['hyperparameters'].map(ast.literal_eval)
model = CNN()
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters() , 0.01)
 train_loss_history = []
test_loss_history = []
train_accuracy_history = []
test_accuracy_history = []
iter = 0
for epoch in range(10):
for i,(images, labels)in enumerate(train_load):
iter += 1
optimizer.zero_grad()
outputs = model(images)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
train_loss, train_accuracy = calculate_loss_accuracy(train_load, model, loss_fn)
 train_loss_history.append(train_loss)
 train_accuracy_history.append(train_accuracy)
 test_loss, test_accuracy = calculate_loss_accuracy(test_load, model, loss_fn)
 test_loss_history.append(test_loss)
 test_accuracy_history.append(test_accuracy)
 print(epoch, 'Done! Train Loss: {}, Test Loss: {}, Train Accuracy {:.2f}%, Test Accuracy {:.2f}%'.format(train_loss, test_loss, train_accuracy, test_accuracy))
view_graphs(train_accuracy_history, test_accuracy_history, train_loss_history, test_loss_history, title='' )<predict_on_test>
hyperparameters = dict(**bayes_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round=10000, early_stopping_rounds=100, metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Bayes Opt.= {:.5f} with std: {:.5f}.'.format(cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )
<predict_on_test><EOS> 
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_bayesian_optimization2.csv', index = False )