Repository: pytorch
Issue: RuntimeError: has_marked_unused_parameters_ INTERNAL ASSERT FAILED at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:290, please report a bug to PyTorch.
Labels: oncall: distributed,triaged

## 🐛 Bug

```
Phase=train
CNN_AE_supervised(
  (encoder): CNN_encoder(
    (conv1): Conv2d(1, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (conv3): Conv2d(16, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  )
  (decoder): CNN_decoder(
    (conv4): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (conv5): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (conv6): Conv2d(16, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  )
  (readout): BN_readout(
    (flatten): Flatten()
    (fcn1): Linear(in_features=504, out_features=24, bias=True)
    (relu1): ReLU()
    (dropout1): Dropout(p=0.3, inplace=False)
    (fcn2): Linear(in_features=24, out_features=1, bias=True)
  )
)
Epoch 0/2
----------
Phase=train
2020-01-22 13:14:16,093 sagemaker-containers ERROR    ExecuteUserScriptError:
Command "/opt/conda/bin/python -m train_ptsrep --backend gloo --epochs 3 --lr 0.01 --seed 42"
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/opt/ml/code/train_ptsrep.py", line 264, in <module>
    train(parser.parse_args())
  File "/opt/ml/code/train_ptsrep.py", line 154, in train
    best_ae_model = dl_procedures.create_and_train_AE_supervised(data_loader, device, dataset_sizes, args.epochs, args.lr, is_distributed, use_cuda)
  File "/opt/ml/code/dl_procedures.py", line 320, in create_and_train_AE_supervised
    num_epochs=num_epochs,model_name="supervised_ae", is_distributed=is_distributed, use_cuda=use_cuda)
  File "/opt/ml/code/dl_procedures.py", line 111, in train_AE_model
    bce_loss.backward()
  File "/opt/conda/lib/python3.6/site-packages/torch/tensor.py", line 118, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/opt/conda/lib/python3.6/site-packages/torch/autograd/__init__.py", line 93, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: has_marked_unused_parameters_ INTERNAL ASSERT FAILED at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:290, please report a bug to PyTorch.
2020-01-22 13:14:16,003 sagemaker-containers ERROR    ExecuteUserScriptError:
Command "/opt/conda/bin/python -m train_ptsrep --backend gloo --epochs 3 --lr 0.01 --seed 42"
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/opt/ml/code/train_ptsrep.py", line 264, in <module>
    train(parser.parse_args())
  File "/opt/ml/code/train_ptsrep.py", line 154, in train
    best_ae_model = dl_procedures.create_and_train_AE_supervised(data_loader, device, dataset_sizes, args.epochs, args.lr, is_distributed, use_cuda)
  File "/opt/ml/code/dl_procedures.py", line 320, in create_and_train_AE_supervised
    num_epochs=num_epochs,model_name="supervised_ae", is_distributed=is_distributed, use_cuda=use_cuda)
  File "/opt/ml/code/dl_procedures.py", line 111, in train_AE_model
    bce_loss.backward()
  File "/opt/conda/lib/python3.6/site-packages/torch/tensor.py", line 118, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/opt/conda/lib/python3.6/site-packages/torch/autograd/__init__.py", line 93, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: has_marked_unused_parameters_ INTERNAL ASSERT FAILED at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:290, please report a bug to PyTorch.

2020-01-22 13:14:18 Uploading - Uploading generated training model
2020-01-22 13:14:48 Failed - Training job failed
```

## To Reproduce

```
def train_AE_model(model, device, dataloaders,dataset_sizes,optimizer,
                   supervised_criterion=None,
                   scheduler=None,
                   num_epochs=25,model_name="default_model", is_distributed=False, use_cuda=False):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    best_loss=1000000000000.0
    finished_batches = 0 

    for epoch in range(num_epochs):
        # Each epoch has a training and validation phase
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        all_labels = {}
        
        for phase in ['train', 'dev']:
            print('Phase={}'.format(phase))
            all_labels[phase]=[]
            
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode. eval mode fixes dropout, batchnorm etc

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            max_batches = 3
            finished_batches = 0
            for (inputs,labels) in dataloaders[phase]: # tqdm(dataloaders[phase])
#                 if finished_batches > max_batches:
#                     break
                
                all_labels[phase].extend(copy.deepcopy(labels))
                inputs = inputs.float().to(device)
                labels = labels.to(device)
                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):

                    
                    if supervised_criterion is not None:
                        # supervised AE
                        outputs,readout = model(inputs)
                        preds = readout.clone().detach().squeeze() > 0.0
                        preds = preds.float()
                        
                        loss = binary_MSE_plus_loss(outputs,inputs,debug=False)
                        bce_loss = supervised_criterion(torch.squeeze(readout), labels.float())
                    else:
                        # plain AE
                        outputs = model(inputs)
                        loss = binary_MSE_plus_loss(outputs,inputs,debug=False)
                        
                    # backward + optimize only if in training phase
                    if phase == 'train':                    
                        if supervised_criterion is not None:
                            loss.backward(retain_graph=True)
                            bce_loss.backward()
                        else:
                            loss.backward()
                        """
                        
                        if is_distributed and not use_cuda:
                        # average gradients manually for multi-machine cpu case only
                            _average_gradients(model)
                        """
                        optimizer.step()
                        

                # statistics
                running_loss += loss.item() * inputs.size(0)
                finished_batches = finished_batches+1
                if supervised_criterion is not None:
                    running_corrects += torch.sum(preds.long() == labels.data)
                
                # Recommended for performance:
                # Flush the pytorch cache
                # https://discuss.pytorch.org/t/why-the-training-slow-down-with-time-if-training-continuously-and-gpu-utilization-begins-to-jitter-dramatically/11444
                if finished_batches % 50 == 0:
                    torch.cuda.empty_cache() 
                del(labels)
                del(inputs)
                finished_batches = finished_batches + 1
            epoch_loss = running_loss #/ dataset_sizes[phase]
            
            if supervised_criterion is not None:
                epoch_acc = running_corrects.double() / dataset_sizes[phase]
                print('{} Loss: {:.4f}, acc={:.4f}'.format(phase, epoch_loss,epoch_acc))
            else:
                print('{} Loss: {:.4f}'.format(phase, epoch_loss))

            # save the best model. Base "best-ness" on dev set loss
            if phase=="dev" and epoch_loss < best_loss:
                best_running_loss = running_loss
                best_model_wts = copy.deepcopy(model.state_dict())
                # save the best model (overwritten each time a better one is found)
                torch.save(best_model_wts,"{}.best_loss.state".format(model_name))
                
            if phase=="dev" and (supervised_criterion is not None) and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_acc_model_wts = copy.deepcopy(model.state_dict())
                # save the best model (overwritten each time a better one is found)
                torch.save(best_model_wts,"{}.best_acc.state".format(model_name))
            
        print()
        # write the model of each epoch as well
        # only for debugging (as these can be large for large networks)
        # torch.save(model.state_dict(), "{}_E{}.state".format(model_name,epoch))

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model
```
```
def create_and_train_AE(data_loader,device,dataset_sizes,num_epochs=5,model_name="plain_ae"):
    # get network sizes
    inputs,labels = iter(data_loader["train"]).next()
    inputs = inputs.float()
    one_data = torch.unsqueeze(inputs[0,:,:],0).float()
    encoder = dl_models.CNN_encoder()
    encoded,encoder_sizes = encoder(one_data,dry_run=True)
    bottleneck_size=np.prod(np.array(encoded.size())[1:])

    # build model proper
    ae = dl_models.CNN_AE(encoder_sizes=encoder_sizes)
    supervised_criterion=torch.nn.BCEWithLogitsLoss()
    optimizer = torch.optim.SGD(ae.parameters(), lr=0.01, momentum=0.9)

    ae.to(device)

    # train model
    best_ae_model = train_AE_model(ae,device,data_loader,dataset_sizes,optimizer,
                   supervised_criterion=None,scheduler=None,
                   num_epochs=num_epochs,model_name=model_name)
``
```

I am trying to train on Sagemaker Pytorch container a supervised convolutional autoenconder (i.e. there is a loss function for the convolutional autoencoder and one for the ANN that comes after it). I get the error above and I cannot figure out what is wrong with my code. It was working before trying to do distributed training. I found this bug report https://github.com/pytorch/pytorch/issues/31035 but in my case .backward() is only called once so it must be something else. Maybe @pietern or someone else can this time also help. 

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @xush6528 @osalpekar