Repository: PaddlePaddle/PaddleSeg

Issue:
使用Lovasz loss 训练时出错
训练一段时间之后突然循环报错以下信息（数据集已经通过脚本校验没有问题）：
```
--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0   std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int)
1   paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int)
2   paddle::platform::GpuMemcpyAsync(void*, void const*, unsigned long, cudaMemcpyKind, CUstream_st*)
3   void paddle::memory::Copy<paddle::platform::CUDAPlace, paddle::platform::CPUPlace>(paddle::platform::CUDAPlace, void*, paddle::platform::CPUPlace, void const*, unsigned long, CUstream_st*)
4   paddle::framework::details::OpHandleBase::RunAndRecordEvent(std::function<void ()> const&)
5   paddle::framework::details::ScaleLossGradOpHandle::RunImpl()
6   paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*)
7   paddle::framework::details::FastThreadedSSAGraphExecutor::RunTracedOps(std::vector<paddle::framework::details::OpHandleBase*, std::allocator<paddle::framework::details::OpHandleBase*> > const&)
8   paddle::framework::details::FastThreadedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&, bool)
9   paddle::framework::details::ScopeBufferedMonitor::Apply(std::function<void ()> const&, bool)
10  paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&, bool)
11  paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&, bool)

----------------------
Error Message Summary:
----------------------
ExternalError:  Cuda error(77), an illegal memory access was encountered.
  [Advise: The device encountered a load or store instruction on an invalid memory address. This leaves the process in an inconsistentstate and any further CUDA work will return the same error. To continue using CUDA, the process must be terminated and relaunched. ] at (/paddle/paddle/fluid/platform/gpu_info.cc:276)
```
配置文件如下：
```
EVAL_CROP_SIZE: (1920, 720) # (width, height), for unpadding rangescaling and stepscaling
TRAIN_CROP_SIZE: (512, 512) # (width, height), for unpadding rangescaling and stepscaling
AUG:
    AUG_METHOD: "stepscaling" # choice unpadding rangescaling and stepscaling
    FIX_RESIZE_SIZE: (1024, 1024) # (width, height), for unpadding
    INF_RESIZE_VALUE: 500  # for rangescaling
    MAX_RESIZE_VALUE: 600  # for rangescaling
    MIN_RESIZE_VALUE: 400  # for rangescaling
    MAX_SCALE_FACTOR: 2.0  # for stepscaling
    MIN_SCALE_FACTOR: 0.5  # for stepscaling
    SCALE_STEP_SIZE: 0.25  # for stepscaling
    MIRROR: True
    RICH_CROP:
        ENABLE: True
        BLUR: True
        BLUR_RATIO: 0.2
        MAX_ROTATION: 90
        BRIGHTNESS_JITTER_RATIO: 0.5
        SATURATION_JITTER_RATIO: 0.5
        CONTRAST_JITTER_RATIO: 0.5
BATCH_SIZE: 2
DATASET:
    DATA_DIR: "./dataset/8156_cut/"
    IMAGE_TYPE: "rgb"  # choice rgb or rgba
    NUM_CLASSES: 2
    TEST_FILE_LIST: "dataset/8156_cut/val.txt"
    TRAIN_FILE_LIST: "dataset/8156_cut/train.txt"
    VAL_FILE_LIST: "dataset/8156_cut/val.txt"
    IGNORE_INDEX: 255
    SEPARATOR: " "
DATALOADER:
    # 数据载入时的并发数, 建议值8
    NUM_WORKERS: 8
    # 数据载入时缓存队列大小, 建议值256
    BUF_SIZE: 256

FREEZE:
    MODEL_FILENAME: "model"
    PARAMS_FILENAME: "params"
MODEL:
    DEFAULT_NORM_TYPE: "bn"
    MODEL_NAME: "deeplabv3p"
    DEEPLAB:
        ASPP_WITH_SEP_CONV: True
        DECODER_USE_SEP_CONV: True
        OUTPUT_STRIDE: 8
TRAIN:
    PRETRAINED_MODEL_DIR: u"pretrained_model/deeplabv3p_xception65_bn_cityscapes"
    MODEL_SAVE_DIR: "saved_model/8156_cut"
    RESUME_MODEL_DIR: ""
    SNAPSHOT_EPOCH: 20
TEST:
    TEST_MODEL: "saved_model/competition/final"
SOLVER:
    LR: 0.01
    LR_POLICY: "poly"
    OPTIMIZER: "sgd"
    NUM_EPOCHS: 100
    LOSS: ["lovasz_hinge_loss","bce_loss"]
    LOSS_WEIGHT:
        LOVASZ_HINGE_LOSS: 0.5
        BCE_LOSS: 0.5
```


Discussion:
可否贴一下报错信息中的python stack
```
Traceback (most recent call last):
  File "pdseg_bk/train.py", line 468, in <module>
    main(args)
  File "pdseg_bk/train.py", line 454, in main
    train(cfg)
  File "pdseg_bk/train.py", line 238, in train
    train_prog, startup_prog, phase=ModelPhase.TRAIN)
  File "/home/aistudio/work/paddleseg/pdseg_bk/models/model_builder.py", line 196, in build_model
    multi_lovasz_softmax_loss(logits, label, mask))
  File "/home/aistudio/work/paddleseg/pdseg_bk/lovasz_losses.py", line 222, in multi_lovasz_softmax_loss
    loss = lovasz_softmax(probas, logit_label, ignore=logit_mask)
  File "/home/aistudio/work/paddleseg/pdseg_bk/lovasz_losses.py", line 133, in lovasz_softmax
    loss = lovasz_softmax_flat(vprobas, vlabels, classes=classes)
  File "/home/aistudio/work/paddleseg/pdseg_bk/lovasz_losses.py", line 166, in lovasz_softmax_flat
    fg_sorted = fluid.layers.gather(fg, perm)
  File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layers/nn.py", line 8328, in gather
    attrs={'overwrite': overwrite})
  File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
    return self.main_program.current_block().append_op(*args, **kwargs)
  File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2917, in append_op
    attrs=kwargs.get("attrs", None))
  File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2123, in __init__
    self.desc.infer_shape(self.block.desc)
paddle.fluid.core_avx.EnforceNotMet: 

--------------------------------------
C++ Traceback (most recent call last):
--------------------------------------
0   paddle::framework::OpDesc::InferShape(paddle::framework::BlockDesc const&) const
1   paddle::operators::GatherOp::InferShape(paddle::framework::InferShapeContext*) const
2   paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
3   paddle::platform::GetCurrentTraceBackString[abi:cxx11]()

----------------------
Error Message Summary:
----------------------
InvalidArgumentError: The last dim of index should be 1 when it is 2D, but we get -1
  [Hint: Expected index_dims[1] == 1, but received index_dims[1]:-1 != 1:1.] (at /paddle/paddle/fluid/operators/gather_op.cc:46)
  [operator < gather > error]
```

Resolving patch:
diff --git a/legacy/pdseg/lovasz_losses.py b/legacy/pdseg/lovasz_losses.py
--- a/legacy/pdseg/lovasz_losses.py
+++ b/legacy/pdseg/lovasz_losses.py
@@ -158,6 +158,10 @@ def lovasz_softmax_flat(probas, labels, classes='present'):
             class_pred = probas[:, 0]
         else:
             class_pred = probas[:, c]
+
+        if len(class_pred.shape) == 1:
+            class_pred = fluid.layers.unsqueeze(class_pred, axes=1)
+
         errors = fluid.layers.abs(fg - class_pred)
         errors_sorted, perm = fluid.layers.argsort(
             errors, axis=0, descending=True)


Test patch: