# Buggy code
def update_core(self):
    """Main update routine of the CustomUpdater."""
    # When we pass one iterator and optimizer to StandardUpdater.__init__,
    # they are automatically named 'main'.
    train_iter = self.get_iterator("main")
    optimizer = self.get_optimizer("main")

    # Get the next batch ( a list of json files)
    batch = train_iter.next()
    self.iteration += 1
    x = self.converter(batch, self.device)

    # Compute the loss at this time step and accumulate it
    if self.ngpu == 0:
        loss = self.model(*x).mean() / self.accum_grad
    else:
        # apex does not support torch.nn.DataParallel
        loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
    if self.use_apex:
        from apex import amp

        with amp.scale_loss(loss, optimizer) as scaled_loss:
            scaled_loss.backward()
    else:
        loss.backward()
    # gradient noise injection
    if self.grad_noise:
        from espnet.asr.asr_utils import add_gradient_noise

        add_gradient_noise(
            self.model, self.iteration, duration=100, eta=1.0, scale_factor=0.55
        )
    loss.detach()  # Truncate the graph

    # update parameters
    self.forward_count += 1
    if self.forward_count != self.accum_grad:
        return
    self.forward_count = 0
    # compute the gradient norm to check if it is normal or not
    grad_norm = torch.nn.utils.clip_grad_norm_(
        self.model.parameters(), self.grad_clip_threshold
    )
    logging.info("grad norm={}".format(grad_norm))
    if math.isnan(grad_norm):
        logging.warning("grad norm is nan. Do not update model.")
    else:
        optimizer.step()
    optimizer.zero_grad()


# Traceback (TypeError)
# asr_train.py --config conf/tuning/train_pytorch_transformer.v3.yaml --preprocess-conf conf/specaug.yaml --ngpu 4 --backend pytorch --outdir exp/train_960_pytorch_train_pytorch_transformer.v3_specaug/results --tensorboard-dir tensorboard/train_960_pytorch_train_pytorch_transformer.v3_specaug --debugmode 1 --dict data/lang_char/train_960_unigram5000_units.txt --debugdir exp/train_960_pytorch_train_pytorch_transformer.v3_specaug --minibatches 0 --verbose 0 --resume --train-json dump/train_960/deltafalse/data_unigram5000.json --valid-json dump/dev/deltafalse/data_unigram5000.json
/work5/t_hayashi/work/espnet_dev/tools/venv/lib/python3.7/site-packages/torch/nn/_reduction.py:49: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
warnings.warn(warning.format(ret))
2019-08-24 01:26:02,701 (asr_train:286) WARNING: Skip DEBUG/INFO messages
Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.

Defaults for this optimization level are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
Processing user overrides (additional kwargs that are not None)...
After processing overrides, optimization options are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
Traceback (most recent call last):
File "/work5/t_hayashi/work/espnet_dev/egs/librispeech/asr1/../../../espnet/bin/asr_train.py", line 352, in <module>
main(sys.argv[1:])
File "/work5/t_hayashi/work/espnet_dev/egs/librispeech/asr1/../../../espnet/bin/asr_train.py", line 339, in main
train(args)
File "/work5/t_hayashi/work/espnet_dev/espnet/asr/pytorch_backend/asr.py", line 425, in train
model, optimizer = amp.initialize(model, optimizer, opt_level=args.train_dtype)
File "/work5/t_hayashi/work/espnet_dev/tools/venv/lib/python3.7/site-packages/apex/amp/frontend.py", line 357, in initialize
return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)
File "/work5/t_hayashi/work/espnet_dev/tools/venv/lib/python3.7/site-packages/apex/amp/_initialize.py", line 164, in _initialize
raise TypeError("optimizers must be either a single optimizer or a list of optimizers.")
TypeError: optimizers must be either a single optimizer or a list of optimizers.

# Fixed code
def update_core(self):
    """Main update routine of the CustomUpdater."""
    # When we pass one iterator and optimizer to StandardUpdater.__init__,
    # they are automatically named 'main'.
    train_iter = self.get_iterator("main")
    optimizer = self.get_optimizer("main")

    # Get the next batch ( a list of json files)
    batch = train_iter.next()
    self.iteration += 1
    x = self.converter(batch, self.device)

    # Compute the loss at this time step and accumulate it
    if self.ngpu == 0:
        loss = self.model(*x).mean() / self.accum_grad
    else:
        # apex does not support torch.nn.DataParallel
        loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
    if self.use_apex:
        from apex import amp

        # NOTE: for a compatibility with noam optimizer
        opt = optimizer.optimizer if hasattr(optimizer, "optimizer") else optimizer
        with amp.scale_loss(loss, opt) as scaled_loss:
            scaled_loss.backward()
    else:
        loss.backward()
    # gradient noise injection
    if self.grad_noise:
        from espnet.asr.asr_utils import add_gradient_noise

        add_gradient_noise(
            self.model, self.iteration, duration=100, eta=1.0, scale_factor=0.55
        )
    loss.detach()  # Truncate the graph

    # update parameters
    self.forward_count += 1
    if self.forward_count != self.accum_grad:
        return
    self.forward_count = 0
    # compute the gradient norm to check if it is normal or not
    grad_norm = torch.nn.utils.clip_grad_norm_(
        self.model.parameters(), self.grad_clip_threshold
    )
    logging.info("grad norm={}".format(grad_norm))
    if math.isnan(grad_norm):
        logging.warning("grad norm is nan. Do not update model.")
    else:
        optimizer.step()
    optimizer.zero_grad()