Repository: PaddlePaddle/models

Issue:
paddle.fluid.core.EnforceNotMet: holder_ should not be null
https://github.com/PaddlePaddle/models/blob/develop/fluid/neural_machine_translation/transformer/train.py
把train.py 参数--use-mem-opt的default改为False￼
报错￼
Traceback (most recent call last):
  File "train.py", line 641, in <module>
    train(args)
  File "train.py", line 593, in train
    token_num, predict, pyreader)
  File "train.py", line 468, in train_loop
    feed=feed_dict_list)
  File "/home/paddle/minqiyang/miniconda2/lib/python2.7/site-packages/paddle/fluid/parallel_executor.py", line 281, in run
    self.executor.run(fetch_list, fetch_var_name)
paddle.fluid.core.EnforceNotMet: holder_ should not be null
Tensor not initialized yet when Tensor::type() is called. at [/paddle/paddle/fluid/framework/tensor.h:141]
PaddlePaddle Call Stacks:
0       0x7f94e3e6e446p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1       0x7f94e3e70a27p paddle::framework::Tensor::type() const + 151
2       0x7f94e44c502bp paddle::operators::ElementwiseOpGrad::GetExpectedKernelType(paddle::framework::ExecutionContext const&) const + 107
3       0x7f94e4ef62bdp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 237
4       0x7f94e4ef2e4cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 252
5       0x7f94e4d4d9a7p
6       0x7f94e4d6b190p
7       0x7f94e4d6a9f5p paddle::framework::details::OpHandleBase::RunAndRecordEvent(std::function<void ()> const&) + 805
8       0x7f94e4d4d47fp paddle::framework::details::ComputationOpHandle::RunImpl() + 95
9       0x7f94e4d6ba95p paddle::framework::details::OpHandleBase::Run(bool) + 117
10      0x7f94e4d429afp
11      0x7f94e3f3da13p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35
12      0x7f94e3f3d1e7p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39
13      0x7f9562d91be0p pthread_once + 80
14      0x7f94e4d41ce2p
15      0x7f94e3f3f284p ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const + 404
16      0x7f9552b6bc5cp
17      0x7f9562d8cdf3p
18      0x7f95623b12cdp clone + 109


Discussion:

Resolving patch:
diff --git a/fluid/neural_machine_translation/transformer/infer.py b/fluid/neural_machine_translation/transformer/infer.py
--- a/fluid/neural_machine_translation/transformer/infer.py
+++ b/fluid/neural_machine_translation/transformer/infer.py
@@ -171,7 +171,7 @@ def fast_infer(test_data, trg_idx2word, use_wordpiece):
         ])
 
     # This is used here to set dropout to the test mode.
-    infer_program = fluid.default_main_program().inference_optimize()
+    infer_program = fluid.default_main_program().clone(for_test=True)
 
     for batch_id, data in enumerate(test_data.batch_generator()):
         data_input = prepare_batch_input(
diff --git a/fluid/neural_machine_translation/transformer/train.py b/fluid/neural_machine_translation/transformer/train.py
--- a/fluid/neural_machine_translation/transformer/train.py
+++ b/fluid/neural_machine_translation/transformer/train.py
@@ -428,7 +428,7 @@ def train_loop(exe, train_prog, startup_prog, dev_count, sum_cost, avg_cost,
     # Since the token number differs among devices, customize gradient scale to
     # use token average cost among multi-devices. and the gradient scale is
     # `1 / token_number` for average cost.
-    build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized
+    # build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized
     train_exe = fluid.ParallelExecutor(
         use_cuda=TrainTaskConfig.use_gpu,
         loss_name=avg_cost.name,


Test patch: