Optuna with Pytorch

Ashish_Verma: <p>Issue =&gt; AttributeError: module ‘optuna.trial’ has no attribute ‘suggest_int’</p>
<h1>
<a name="code-1" class="anchor" href="#code-1"></a>Code:</h1>
<h1>
<a name="coding-utf-8-2" class="anchor" href="#coding-utf-8-2"></a>-<em>- coding: utf-8 -</em>-</h1>
<p>“”"<br>
Created on Mon May 15 20:22:07 2023</p>
<p><span class="mention">@author</span>: Ashish<br>
“”"</p>
<p>import torch<br>
import numpy as np<br>
from optuna.trial import TrialState<br>
import pandas as pd<br>
from  torch.utils.data import Dataset,DataLoader<br>
import optuna<br>
from sklearn.model_selection import train_test_split<br>
torch.manual_seed(42)</p>
<p>CLASSES = 1<br>
BATCH_SIZE = 32<br>
EPOCHS = 100<br>
DEVICE =‘gpu’<br>
N_TRAIN_EXAMPLES = BATCH_SIZE * 30<br>
N_VALID_EXAMPLES = BATCH_SIZE * 10</p>
<p>class BostonData(Dataset):</p>
<pre><code>def __init__(self,X,y):
    self.X = torch.from_numpy(X)
    self.y = torch.from_numpy(y)

def __len__(self):
    return len(self.X)

def __getattr__(self,item):
    return self.X[item],self.y[item]
</code></pre>
<p>def get_model(trail,in_features,n_layers,dropout,n_output):<br>
layers = []<br>
fc_layer = in_features<br>
for i in range(n_layers):<br>
out_features = trial.suggest_int(“n_units_l{}”.format(i),2,in_features)<br>
layers.append(torch.nn.Linear(in_features, out_features))<br>
layers.append(torch.nn.LeakyReLU())<br>
in_features = out_features</p>
<pre><code>layers.append(torch.nn.Linear(in_features, fc_layer)) 
layers.append(torch.nn.LeakyReLU())

layers.append(torch.nn.Dropout(dropout))
layers.append(torch.nn.Linear(fc_layer,n_output)) 
   
return torch.nn.Sequential(*layers)
</code></pre>
<p>def objective(trial):</p>
<pre><code># For TPESampler 
params = {
          'learning_rate': trial.suggest_loguniform('learning_rate', 1e-6, 1e-2), 
          'optimizer': trial.suggest_categorical("optimizer", ["Adam", "RMSprop", "SGD"]),
          'weight_decay': trial.suggest_loguniform('weight_decay', 1e-4, 1e-2),
          "n_layers" : trial.suggest_int("n_layers", 1, 4),
          "dropout" : trial.suggest_float('dropout',0.1,0.5,step = 0.1)
          }
test_loss   = train_net(trial, params) 
return np.mean(test_loss)
</code></pre>
<p>def train_net(trial, params):</p>
<pre><code>data_url = "http://lib.stat.cmu.edu/datasets/boston"
raw_df = pd.read_csv(data_url, sep="\s+", skiprows=22, header=None)
X = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])
y = raw_df.values[1::2, 2]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)

# Prepare Boston dataset
trainloader = DataLoader(BostonData(X_train, y_train), batch_size=BATCH_SIZE, shuffle=True, num_workers=1)
valloader = DataLoader(BostonData(X_val, y_val), batch_size=1, shuffle=False, num_workers=1)

# Initialize the MLP
net = get_model(trial,
        in_features= X.shape[1],
        n_layers=params['n_layers'] ,
        dropout=params['dropout'],
        n_output= 1)
</code></pre>
<h1>
<a name="define-the-loss-function-and-optimizer-3" class="anchor" href="#define-the-loss-function-and-optimizer-3"></a>Define the loss function and optimizer</h1>
<pre><code>loss_function = torch.nn.MSELoss()
optimizer = getattr(torch.optim, params['optimizer'])(net.parameters(), lr= params['learning_rate'], weight_decay=params['weight_decay'])

# Run the training loop
for epoch in range(0, 5): # 5 epochs at maximum
    
    total_test_loss = []
    # Print epoch
    print(f'Starting epoch {epoch+1}')
    
    # Set current loss value
    #current_loss = 0.0
    
    # Iterate over the DataLoader for training data
    for i, data in enumerate(trainloader, 0):
    
        # Get and prepare inputs
        inputs, targets = data
        inputs, targets = inputs.float(), targets.float()
        targets = targets.reshape((targets.shape[0], 1))
        
        # Zero the gradients
        optimizer.zero_grad()
        # Perform forward pass
        outputs = net(inputs)
        # Compute loss
        loss = loss_function(outputs, targets)
        # Perform backward pass
        loss.backward()
        # Perform optimization
        optimizer.step()
        
    net.eval()
    with torch.no_grad():
        for i, data in enumerate(valloader, 0):
            # Get and prepare inputs
            inputs, targets = data
            inputs, targets = inputs.float(), targets.float()
            targets = targets.reshape((targets.shape[0], 1))
            
            # # Perform forward pass
            test_outputs = net(inputs)
            test_loss = loss_function(test_outputs, targets)
            total_test_loss.append(test_loss.item())
    

# Process is complete.
return total_test_loss
</code></pre>
<p>study = optuna.create_study(direction =“minimize”, sampler=optuna.samplers.TPESampler())<br>
study.optimize(objective, n_trials= 100,timeout=600)</p>
<p>pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED])<br>
complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])<br>
print("Study statistics: “)<br>
print(”  Number of finished trials: “, len(study.trials))<br>
print(”  Number of pruned trials: “, len(pruned_trials))<br>
print(”  Number of complete trials: ", len(complete_trials))</p>
<p>print(“Best trial:”)<br>
trial = study.best_trial</p>
<p>print("  Value: ", trial.value)</p>
<h1>
<a name="print-params-for-key-value-in-trialparamsitems-print-formatkey-value-4" class="anchor" href="#print-params-for-key-value-in-trialparamsitems-print-formatkey-value-4"></a>print("  Params: “)<br>
for key, value in trial.params.items():<br>
print(”    {}: {}".format(key, value))</h1>
<p>Any help is appretiated!!!</p>

ptrblck: <p><code>optuna.trial.Trial.suggest_int</code> is a valid method as seen <a href="https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_int">here</a> so you might need to update <code>optuna</code> in case it’s not available in your release.</p>