Gradient accumulation and distributed parallelism will reduce the effect?
Author: wulaoshi
Pull request: no
Labels: name: wontfix

Reference the code: "transformers/examples/legacy/question-answering/run_squad.py"
I found：
1. normal results without using distributed code, using only gradient accumulation.
2. using the distributed code with the parameter gradient_accumulation_steps=1, the effect is normal.
3. using the distributed code with the parameter gradient_accumulation_steps set to other, the experimental effect is abnormal.

What is going on, please? Thanks!


Comments:
The following is part of the reference code after my deletion:

```
def get_binary_data(args, macro, tokenizer, read_data=False):
    if macro.local_rank not in [-1, 0]:
        torch.distributed.barrier()

    train_file = ["a.txt", "b.txt", "c.txt"]
    valid_file = ["a_valid.txt", "b_valid.txt", "c_valid.txt"]
    train_data, vaild_data = [],[]
    cached_features_file = args.train_path+"utt_generator"
    if read_data == False:
        for t_f in train_file:
            train_utterance = file_reader(args.train_path+t_f)
            datasets = get_data_loaders(train_utterance, tokenizer)
            train_data.extend(datasets)
        for t_f in valid_file:
            vaild_utterance = file_reader(args.train_path+t_f)
            datasets = get_data_loaders(vaild_utterance, tokenizer)
            vaild_data.extend(datasets)
    else:
        read_data = torch.load(cached_features_file)
        train_data = read_data["train_data"]
        vaild_data = read_data["dev_data"]

    train_len = len(train_data)
    logger.info("train len:%d, valid len:%d."%(len(train_data), len(vaild_data)))

    train_batch_size = args.batch_size * max(1, macro.n_gpu)
    train_sampler = RandomSampler(train_data) if macro.local_rank == -1 else DistributedSampler(train_data)
    train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=train_batch_size, collate_fn=collate_fn)

    eval_batch_size = args.batch_size * max(1, macro.n_gpu)
    eval_sampler = SequentialSampler(vaild_data)
    eval_dataloader = DataLoader(vaild_data, sampler=eval_sampler, batch_size=eval_batch_size, collate_fn=collate_fn_test)

    if macro.local_rank == 0:
        torch.distributed.barrier()
    return train_dataloader, eval_dataloader, train_len

def train(model, training_data, optimizer, device, scheduler, args, macro):
    model.train()
    batch_idx = 0
    epoch_loss = 0
    logging_loss = 0
    global global_step
    global tb_writer
    for batch in tqdm(
        training_data,
        mininterval=2,
        desc=" - (Traning) ",
        leave=False,
        disable=macro.local_rank not in [-1, 0]
    ):
        batch_idx += 1
        input_ids, lm_labels, token_type_ids, attention_mask = list(map(lambda x: x.to(device), batch))
        (lm_loss), *_ = model(input_ids, token_type_ids=token_type_ids, lm_labels=lm_labels, attention_mask=attention_mask)

        if macro.n_gpu > 1:
            lm_loss = lm_loss.mean()  # mean() to average on multi-gpu parallel (not distributed) training
        loss = lm_loss / args.gradient_accumulation_steps
        loss.backward()
        epoch_loss += loss.item()
        if (batch_idx) % args.gradient_accumulation_steps == 0:
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            optimizer.zero_grad()
            scheduler.step()
    return epoch_loss/batch_idx

@func_time
def train_epoch(model, train_data, dev_data, optimizer, device, scheduler, best_result, args, tokenizer, epoch, macro):
    if macro.local_rank in [-1, 0]:
        logger.info("----" * 5)
        logger.info('Epoch: {}'.format(epoch))
    train_loss = train(model, train_data, optimizer, device, scheduler, args, macro)
    if epoch>=12:
        if macro.local_rank in [-1, 0]:
            model_to_evaluate = model.module if hasattr(model, "module") else model
            vaild_bleu = evaluate(model_to_evaluate, dev_data, device, tokenizer, args)
            if best_result < vaild_bleu:
                best_result = vaild_bleu
                torch.save(model_to_evaluate.state_dict(), args.output_model_path + "_valid")
                logger.info('save:{}'.format(args.output_model_path + "_valid"))
            logger.info("Val. Bleu: %4f" % (vaild_bleu))
    if epoch%5==0:
        if macro.local_rank == -1 or torch.distributed.get_rank() == 0:
            model_to_save = model.module if hasattr(model, "module") else model
            torch.save(model_to_save.state_dict(), args.output_model_path+"_"+str(epoch))
            logger.info('save:{}'.format(args.output_model_path+"_"+str(epoch)))
    if macro.local_rank in [-1, 0]:
        logger.info("Train Loss:%.5f"%(train_loss))
        logger.info("----"*5)
    return best_result

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--no_cuda', action="store_true")
    parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus")
    macro = parser.parse_args()
    args = OptionSet()

    args.model_name = "test"
    args.model_name = args.model_name+"_distributed_nogrid"

    global logger
    logger = create_logger(args)

    # Setup CUDA, GPU & distributed training
    if macro.local_rank == -1 or macro.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not macro.no_cuda else "cpu")
        macro.n_gpu = 0 if macro.no_cuda else torch.cuda.device_count()
    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.cuda.set_device(macro.local_rank)
        device = torch.device("cuda", macro.local_rank)
        torch.distributed.init_process_group(backend="nccl")
        macro.n_gpu = 1
    # args.device = device
    set_seed(macro, args.seed)
    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO if macro.local_rank in [-1, 0] else logging.WARN,
    )
    logger.warning(
        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
        macro.local_rank,
        device,
        macro.n_gpu,
        bool(macro.local_rank != -1),
        # args.fp16,
    )
    # Load pretrained model and tokenizer
    if macro.local_rank not in [-1, 0]:
        # Make sure only the first process in distributed training will download model & vocab
        torch.distributed.barrier()

    logger.info('using device:{}'.format(device))
    model, _, tokenizer = creat_model()
    if macro.local_rank == 0:
        # Make sure only the first process in distributed training will download model & vocab
        torch.distributed.barrier()
    model = model.to(device)
    global PAD_idx
    PAD_idx = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[-3])
    train_data, dev_data, train_len = get_binary_data(args, macro, tokenizer, read_data=False)
    # Training phase.
    logger.info("Start training.")
    instances_num = train_len
    train_steps = int(instances_num * args.epochs_num / args.batch_size) + 1

    logger.info('Batch size: {}'.format(args.batch_size))
    logger.info('The number of training instances:{}'.format(instances_num))

    num_parameters = 0
    parameters = model.parameters()
    for parameter in parameters:
        num_parameters += parameter.numel()
    logger.info('number of model parameters: {}'.format(num_parameters))

    decoder_layer = list(map(id, model.decoder.layers.parameters()))
    encoder_para = filter(lambda p: id(p) not in (decoder_layer), model.parameters())
    optimizer_grouped_parameters = [
        {'params': encoder_para, 'lr': args.learning_rate, 'weight_decay_rate': 0.01},
        {'params': model.decoder.layers.parameters(), 'lr': args.learning_rate * 5, 'weight_decay_rate': 0.01}
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, correct_bias=False)
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=train_steps*args.warmup, num_training_steps=train_steps)

    # multi-gpu training (should be after apex fp16 initialization)
    if macro.n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # Distributed training (should be after apex fp16 initialization)
    if macro.local_rank != -1:
        model = torch.nn.parallel.DistributedDataParallel(
            model, device_ids=[macro.local_rank], output_device=macro.local_rank, find_unused_parameters=True
        )

    best_result = 0.0
    for epoch in range(1, args.epochs_num+1):
        best_result = train_epoch(model, train_data, dev_data, optimizer, device, scheduler, best_result, args, tokenizer, epoch, macro)

```

This is the result of my experiment:
![image](https://user-images.githubusercontent.com/27938964/106379879-ed052800-63e9-11eb-9f2e-bbbc96d6f12e.png)


This issue has been automatically marked as stale and been closed because it has not had recent activity. Thank you for your contributions.

If you think this still needs to be addressed please comment on this thread.