Hyperparameter optimization (Grid Search, Bayesian Optimimzation, Tree-structured Parzen Estimators) 

##Reference
[Algorithms for Hyper-Parameter Optimization](https://www.lri.fr/~kegl/research/PDFs/BeBaBeKe11.pdf)

[Hyperparameter optimization for Neural Networks](http://neupy.com/2016/12/17/hyperparameter_optimization_for_neural_networks.html)

[ハイパーパラメータ自動最適化ツール「Optuna」公開](https://research.preferred.jp/2018/12/optuna-release/)

##Library
```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from mpl_toolkits.mplot3d import Axes3D

#import math

#import tensorflow as tf

import optuna
from bayes_opt import BayesianOptimization

from keras.datasets import mnist

import xgboost as xgb
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import accuracy_score
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

from sklearn.model_selection import GridSearchCV
```

##Exapmle 1
```python
def f(x):
  y = 2*x**4 - 5*x**2 + x
  return y

plt.figure(figsize=(5, 3))
X = [x for x in np.arange(-2, 2, 0.1)]
y = [f(x) for x in X]
plt.plot(X, y)
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.show()
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/abecc38e-fdce-3141-ba85-b165ad4981d8.png)


```python
# Bayesian Optimization
def f_bo(x):
  y = 2*x**4 - 5*x**2 + x
  return -y

pbounds = {'x': (-2, 2)}

bo = BayesianOptimization(f=f_bo, pbounds=pbounds, random_state=100)
bo.maximize(init_points=3, n_iter=3)

for i, res in enumerate(bo.res):
    print('Iteration {}: \n\t{}'.format(i, res))
    
print (bo.max)    

X = [x for x in np.arange(-2, 2, 0.1)]
y = [f(x) for x in X]

plt.figure(figsize=(5, 3))
plt.plot(X, y, label='f(x)')

xs = bo.max['params']['x']
ys = bo.max['target']
plt.plot(xs, -ys, "ro", label='solution')

plt.title('BayesianOptimization')
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.legend(loc='best')
plt.show()
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/324efd8d-0be0-eadf-490e-3cd8b789d379.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/4b7f7782-b48b-7c42-0a7a-ff4aeeba09b9.png)

```python
#Optuna
def objective(trial):
  x = trial.suggest_uniform('x', -2, 2)
  y = 2*x**4 - 5*x**2 + x
  return y

study = optuna.create_study()
study.optimize(func=objective, n_trials=10)

print (len(study.trials))
print ("minimum f(x) : ", study.best_value)
print ("argmin f(x) : ", study.best_params)

X = [x for x in np.arange(-2, 2, 0.1)]
y = [f(x) for x in X]

plt.figure(figsize=(5, 3))
plt.plot(X, y, label='f(x)')

xs = study.best_params['x']
ys = study.best_value
plt.plot(xs, ys, 'ro', label='solution')

plt.title('Optuna (TPE)')
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.legend(loc='best')
plt.show()
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/0d915457-8561-2429-3e70-d97765646e0b.png)


##Exapmle 2
```python
def f(x1, x2):
  sigma = 4.0
  y = np.exp(-(x1**2 + x2**2)/(2*sigma**2)) / (2*np.pi*sigma**2)
  
  return y

x1 = np.arange(-3.0, 3.0, 0.1)
x2 = np.arange(-3.0, 3.0, 0.1)

X1, X2 = np.meshgrid(x1, x2)
y = f(X1, X2)

fig = plt.figure(figsize=(5, 3))
ax = fig.add_subplot(111, projection='3d')

#surf = ax.plot_surface(X1, X2, y, cmap='bwr', linewidth=0)
#fig.colorbar(surf)

#ax.plot_surface(X1, X2, y)
ax.plot_wireframe(X1, X2, y)

#ax.set_title("Surface Plot")

plt.show()
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/4534dcdc-b21d-c84f-9df7-53f89129bcf4.png)


```python
#Bayesian Optimization
def f_bo(x1, x2):
  sigma = 4.0
  y = np.exp(-(x1**2 + x2**2)/(2*sigma**2)) / (2*np.pi*sigma**2)

  return y

pbounds = {'x1': (-2, 2), 'x2': (-2, 2)}

bo = BayesianOptimization(f=f_bo, pbounds=pbounds, random_state=100)
bo.maximize(init_points=3, n_iter=5)

x1 = np.arange(-3.0, 3.0, 0.1)
x2 = np.arange(-3.0, 3.0, 0.1)

X1, X2 = np.meshgrid(x1, x2)
y = f(X1, X2)

fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111, projection='3d')
ax.plot_wireframe(X1, X2, y)

xs = [bo.max['params']['x1']]
ys = [bo.max['params']['x2']]
zs = [bo.max['target']]

ax.scatter(xs, ys, zs, label='solution', c='r', marker='o')

ax.set_title('Bayesian Optimization')

plt.show()
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/0f1d1d9c-9df5-f65d-1914-41410d985d31.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/0f9a5c9e-b0f5-e329-cf36-c08cad98acdb.png)


```python
#Optuna
def objective(trial):
  x1 = trial.suggest_uniform('x1', -2, 2)
  x2 = trial.suggest_uniform('x2', -2, 2)
  
  sigma = 4.0
  y = np.exp(-(x1**2 + x2**2)/(2*sigma**2)) / (2*np.pi*sigma**2)
  return -y

study = optuna.create_study()
study.optimize(func=objective, n_trials=5)

print (len(study.trials))
print ("minimum f(x) : ", study.best_value)
print ("argmin f(x) : ", study.best_params)

x1 = np.arange(-3.0, 3.0, 0.1)
x2 = np.arange(-3.0, 3.0, 0.1)

X1, X2 = np.meshgrid(x1, x2)
y = f(X1, X2)

fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111, projection='3d')
ax.plot_wireframe(X1, X2, y)

xs = [study.best_params['x1']]
ys = [study.best_params['x2']]
zs = [-study.best_value]
ax.scatter(xs, ys, zs, label='solution', c='r', marker='o')

ax.set_title('Optuna (TPE)')

plt.show()

```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/5f058bcc-0b78-58f4-edcf-d5c42042a283.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/b8c7ae97-adaf-e255-cded-7e5c2ddd647b.png)


##Exapmle 3 (MNIST, RandomForest, XGBoost)
```python
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)

x_train = X_train.astype('float32')
x_test = X_test.astype('float32')
x_train /= 255
x_test /= 255

x_train = x_train[:1000]
x_test = x_test[:500]
y_train = y_train[:1000]
y_test = y_test[:500]

print ('x_train shape: ', x_train.shape)
print ('y_test shape: ', y_test.shape)
```

```python
Xgb = xgb.XGBClassifier()
Xgb.fit(x_train,y_train)

xgb_predict = Xgb.predict(x_test)

print("Default XGBoost")
print('Accuracy test: ', accuracy_score(y_test, xgb_predict))
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/5cd121e8-ead5-cad3-ce2c-7f0a996c934f.png)

```python
rf = RandomForestClassifier()
rf.fit(x_train,y_train)

rf_predict = rf.predict(x_test)

print("Default Random Forest")
print('Accuracy test: ', accuracy_score(y_test, rf_predict))

kfold = KFold(n_splits=10, random_state=7)
results = cross_val_score(rf, x_train, y_train, cv=kfold)

score = results.mean()*100

print ('Scores: ', results)
print('Accuracy (mean): %.2f%%' % (results.mean()*100))

```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/f3d1ff77-f4e8-601e-babc-6d2256d81b1a.png)


![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/712ade0e-2c75-45e1-275f-1c278189ed10.png)

```python
#Grid search
param_grid = {'n_estimators': [5, 10, 15],  'criterion': ['gini', 'entropy']}

grid_search = GridSearchCV(rf, param_grid, cv=5)
grid_search.fit(x_train, y_train)

print('Test set score: {}'.format(grid_search.score(x_test, y_test)))
print('Best parameters: {}'.format(grid_search.best_params_))
print('Best cross-validation: {}'.format(grid_search.best_score_))
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/fab2407a-fa9d-80e1-291c-753aee163f66.png)

```python
#Bayesian optimazation
def randomforest_cv(n_estimators, min_samples_split, max_features):
  val = cross_val_score(
      RandomForestClassifier(n_estimators=int(n_estimators), 
                             min_samples_split=int(min_samples_split),
                             max_features=max_features, 
                             random_state=0),
      x_train, y_train,
      scoring = 'accuracy',
      cv = 3, # 3-fold
      n_jobs = 1).mean()
  return val

bo = BayesianOptimization(
    randomforest_cv,
    {'n_estimators': (10, 250),
    'min_samples_split': (2, 25),
    'max_features': (0.1, 0.999)})

bo.maximize(init_points=3, n_iter=5)
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/995e697a-0afc-0818-a081-e7c30dd0bede.png)

```python
print ('best max_features: ', bo.max['params']['max_features'])
print ('best min_samples_split: ', bo.max['params']['min_samples_split'])
print ('best n_estimators: ', bo.max['params']['n_estimators'])
print ('max accuracy test : ', bo.max['target'])
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/ba686f48-eba3-d0de-debe-4be288d50dd6.png)


```python
#Optuna
def objective_rf_cv(trial):
  # Categorical parameter
  criterion = trial.suggest_categorical('criterion', ['gini', 'entropy'])

  # Int parameter
  n_estimators = trial.suggest_int('n_estimators', 10, 30)

  # Uniform parameter
  max_features = trial.suggest_uniform('max_features', 0.1, 0.3)

  # Loguniform parameter
  #learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e-2)

  # Discrete-uniform parameter
  #drop_path_rate = trial.suggest_discrete_uniform('drop_path_rate', 0.0, 1.0, 0.1)
    
  val = cross_val_score(
      RandomForestClassifier(n_estimators=n_estimators, 
                             criterion=criterion,
                             max_features=max_features, 
                             random_state=0),
      x_train, y_train,
      scoring = 'accuracy',
      cv = 3, # 3-fold
      n_jobs = 1).mean()
  return -val

study = optuna.create_study()
study.optimize(func=objective_rf_cv, n_trials=10)

print ('best n_estimators: ', study.best_params['n_estimators'])
print ('best criterion: ', study.best_params['criterion'])
print ('best max_features: ', study.best_params['max_features'])
print ('max accuracy test : ', -study.best_value)
```
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/6148aa61-0f23-0e30-477a-87dd74ba60eb.png)


![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/d9db3fe1-4557-c902-68c6-af8b72a1f9f1.png)


##Exapmle 4 (MNIST, CNN)
```python
(x_train, y_train),(x_test, y_test) = tf.keras.datasets.mnist.load_data()

x_train = x_train.astype(float) / 255.0
x_test = x_test.astype(float) / 255.0

x_train = x_train.reshape(-1, 28, 28, 1)
x_test = x_test.reshape(-1, 28, 28, 1)

num_classes = 10
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)

print ("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)
print ("x_test shape:", x_test.shape, "y_test shape:", y_test.shape)

x_train = x_train[:20000]
x_test = x_test[:3000]
y_train = y_train[:20000]
y_test = y_test[:3000]

print ("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)
print ("x_test shape:", x_test.shape, "y_test shape:", y_test.shape)
```

```python
#Optuna
def objective(trial):
  
  dropout_rate = trial.suggest_uniform('dropout_rate', 0.0, 0.5)
  
  batch_size=trial.suggest_categorical('batch_size', [256, 512])
  
  # Loguniform parameter
  learning_rate = trial.suggest_loguniform('learning_rate', 1e-4, 1e-2)
  
  model = tf.keras.Sequential()

  # Must define the input shape in the first layer of the neural network
  model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', 
                                   activation='relu', input_shape=(28,28,1))) 
  model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
  #model.add(tf.keras.layers.Dropout(0.3))
  model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', 
                                   activation='relu'))
  model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
  #model.add(tf.keras.layers.Dropout(0.3))
  model.add(tf.keras.layers.Flatten())
  model.add(tf.keras.layers.Dense(256, activation='relu'))
  model.add(tf.keras.layers.Dropout(dropout_rate))
  model.add(tf.keras.layers.Dense(10, activation='softmax'))
  
  model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
  
  model.fit(x_train,
         y_train,
         batch_size=batch_size,
         epochs=3,
         validation_data=(x_test, y_test))
  
  score = model.evaluate(x_test, y_test, verbose=0)
  
  return 1.0 - score[1]   
```

```python
study = optuna.create_study()
study.optimize(objective, n_trials=10)

print ("best params: ", study.best_params)
print ("best test accuracy: ", 1 - study.best_value)
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/c221ace1-1710-70a9-2905-d8a133a3a9fe.png)


![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/c0a6362b-81e3-ef76-50d5-65083dd1b18a.png)