Optuna PyTorch integration error

katie-bw: <p>Hello,<br>
I’ve been trying to implement Optuna into my Pytorch workflow for hyperparameter optimization with the code below:</p>
<pre><code class="lang-auto">from optuna import trial 
from optuna.trial import TrialState

def define_model(trial): 
    in_features = torch.tensor(1024, requires_grad = False)
    n_layers = trial.suggest_int('n_layers', 1, 8)
    #Suggest a number of layers from 1 to 8
    net = nn.Sequential()
    #Define the number of input features 
    for i in range(n_layers): 
        #Loop through the number of proposed layers 
        out_features = torch.tensor((trial.suggest_int('n_units_l{}'.format(i), 1, in_features)))
        #Define the number of output features between the layers
        net.append(nn.Linear(in_features, out_features, dtype = torch.float64))
        #Add a linear layer to the NN 
        net.append(nn.ReLU())
        #Add the activation function 
        net.append(nn.Dropout(torch.tensor(trial.suggest_float('dropout_l{}'.format(i), 0.1, 0.5))))
        #Define the dropout rate 
        #in_features = torch.tensor(out_features)
        in_features = out_features.clone().detach()
        #Redefine the number of input features for the next layer 
        return in_features
    net.append(nn.Linear(in_features, 1, dtype = torch.int8))
    #Define output layer     
    return net

def objective(trial): 
    model = define_model(trial) 
    #Define the model 
    optimizer_name = trial.suggest_categorical('optimizer', ['Adam', 'Adamax', 'ASGD', 'LBFGS', 'SGD'])
    #Define the optimizer
    lr = trial.suggest_float('lr', 1e-5, 1e-1, log = True)
    #Define the learning rate
    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr = lr)
    #Define the optimizer 
    criterion = nn.MSELoss()
    #Define the criterion
    epochs = trial.suggest_int('epochs', 50, 500, step = 50)
    #Define the number of epochs 

    #Model training
    train_time = time.time()
    model.train()
    for epoch in range(epochs): 
        running_loss = 0 
        for data, target in enumerate(train_loader):
            #Loop through the training set
            optimizer.zero_grad()
            #Reset the gradients 
            output = model(data)
            #Run the model on the data 
            loss = criterion(output, target)
            #Calculate the loss 
            loss.backward()
            optimizer.step()
            #Backpropogate
            running_loss += loss.item()
            #Append to running loss
        #Model validation
        model.eval()
        #Change to evaluation mode 
        y_pred_test = model(X_test)
        #Run model on the train 
        test_mse = torch.mean((y_test - y_pred_test) **2).item()
        trial.report(test_mse, epoch)
        if trial.should_prune():
            raise optuna.exceptions.TrialPruned()
    return test_mse

in_features = 1024
train_loader = train_loader_conc_1024
y_train = y_train_conc_1024
X_train = X_train_conc_1024
y_test = y_test_conc_1024
X_test = X_test_conc_1024
study = optuna.create_study(direction = 'minimize')
study.optimize(objective, n_trials = 500, timeout = 1200)

pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED])
complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])

print("Study statistics: ")
print("  Number of finished trials: ", len(study.trials))
print("  Number of pruned trials: ", len(pruned_trials))
print("  Number of complete trials: ", len(complete_trials))

print("Best trial:")
trial = study.best_trial

print("  Value: ", trial.value)

print("  Params: ")
for key, value in trial.params.items():
    print("    {}: {}".format(key, value))
</code></pre>
<p>But I’m getting the following error,</p>
<pre><code class="lang-auto">AttributeError                            Traceback (most recent call last)
/home/kvb32/Documents/CYP450/CYP450/small_model/model_building.ipynb Cell 14 in &lt;cell line: 78&gt;()
     76 X_test = X_test_conc_1024
     77 study = optuna.create_study(direction = 'minimize')
---&gt; 78 study.optimize(objective, n_trials = 500, timeout = 1200)
     80 pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED])
     81 complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])

File ~/anaconda3/envs/my-rdkit-env/lib/python3.10/site-packages/optuna/study/study.py:425, in Study.optimize(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)
    321 def optimize(
    322     self,
    323     func: ObjectiveFuncType,
   (...)
    330     show_progress_bar: bool = False,
    331 ) -&gt; None:
    332     """Optimize an objective function.
    333 
    334     Optimization is done by choosing a suitable set of hyperparameter values from a given
   (...)
    422             If nested invocation of this method occurs.
    423     """
--&gt; 425     _optimize(
    426         study=self,
    427         func=func,
...
---&gt; 35 optimizer = getattr(optim, optimizer_name)(model.parameters(), lr = lr)
     36 #Define the optimizer 
     37 criterion = nn.MSELoss()

AttributeError: 'Tensor' object has no attribute 'parameters'
</code></pre>
<p>Any assistance would be greatly appreciated <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=12" title=":slight_smile:" class="emoji" alt=":slight_smile:" loading="lazy" width="20" height="20"> (datasets and loaders are generated in a previous block and seem to be working fine)</p>

ptrblck: <p>Based on the error message it seems <code>model</code> is a tensor while an <code>nn.Module</code> is expected. Could you check what <code>model = define_model(trial)</code> returns and make sure it’s indeed a real model?</p>

katie-bw: <p>thank you for the response! It is returning a tensor, I’ve changed it slightly by initializing net as a list <code>net = [nn.Flatten()]</code> and then returning as <code>nn.Sequential(*net)</code> but I’m still getting a tensor rather than a nn.Module how would I fix this?</p>

ptrblck: <p>Your new code snippet should now be a module, but would not contain any trainable parameters since <code>nn.Flatten</code> will just change the shape of the input activation without any training.</p>
<pre><code class="lang-python">net = [nn.Flatten()]
model = nn.Sequential(*net)

optimizer = torch.optim.SGD(model.parameters(), lr=1.)
# ValueError: optimizer got an empty parameter list
</code></pre>
<p>Could you explain why you want to initialize an optimizer if no model seems to be available?</p>