Issue 1:
Please make sure that this is a bug. As per our GitHub Policy, we only address code/doc bugs, performance issues, feature requests and build/installation issues on GitHub. tag:bug_template

System information

Have I written custom code (as opposed to using a stock example script provided in TensorFlow): Yes

OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Linux version 3.10.0-957.10.1.el7.x86_64 (mockbuild@x86-040.build.eng.bos.redhat.com) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) )

Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device: N/A

TensorFlow installed from (source or binary): binary pip install

TensorFlow version (use command below): v1.13.1-0-g6612da8951 1.13.1

Python version: 3.6.4

Bazel version (if compiling from source): N/A

GCC/Compiler version (if compiling from source): N/A

CUDA/cuDNN version: N/A (CPU)

GPU model and memory: N/A (CPU)

Describe the current behavior

Error occurs when trying to compute the Jacobian in the following code

Describe the expected behavior

<tf.Tensor: id=XXX, shape=(2, 1), dtype=float32, numpy=array([[1.], [1.]], dtype=float32)>

Code to reproduce the issue

Provide a reproducible test case that is the bare minimum necessary to generate the problem.

import tensorflow as tf
tf.enable_eager_execution()

with tf.GradientTape() as g:
x = tf.constant([1.0])
g.watch(x)
y = tf.concat([x,x], axis=0)

g.jacobian(y, x)

Other info / logs

Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.

Additional info: the expected behavior occurs when creating the GradientTape with persistent=True and replacing the last line of code with g.jacobian(y, x, experimental_use_pfor=False). However, that workaround is extremely slow with real data.

Log of the code that reproduces the issue:

AttributeError Traceback (most recent call last)
in ()
9 # y = tf.concat([y,y], axis=0)
10
---> 11 g.jacobian(y, x)

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/backprop.py in jacobian(self, target, sources, unconnected_gradients, parallel_iterations, experimental_use_pfor)
1021 try:
1022 output = pfor_ops.pfor(loop_fn, target_size,
-> 1023 parallel_iterations=parallel_iterations)
1024 except ValueError as err:
1025 six.reraise(

/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/parallel_for/control_flow_ops.py in pfor(loop_fn, iters, parallel_iterations)
149 if context.executing_eagerly():
150 f = function.defun(f)
--> 151 return f()
152
153

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py in call(self, *args, **kwargs)
862 def call(self, *args, **kwargs):
863 """Calls a graph function specialized to the inputs."""
--> 864 graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
865 return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
866

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
1174 self._input_signature,
1175 autograph=self._autograph,
-> 1176 arg_names=arg_names),
1177 self._function_attributes)
1178 if self._input_signature:

/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, add_control_dependencies, arg_names, op_return_value)
446 tf_decorator.rewrap(python_func, original_func, converted_func)
447
--> 448 func_outputs = python_func(*func_args, **func_kwargs)
449
450 # invariant: func_outputs contains only Tensors, IndexedSlices,

/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/parallel_for/control_flow_ops.py in f()
146 """
147 def f():
--> 148 return _pfor_impl(loop_fn, iters, parallel_iterations=parallel_iterations)
149 if context.executing_eagerly():
150 f = function.defun(f)

/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/parallel_for/control_flow_ops.py in _pfor_impl(loop_fn, iters, parallel_iterations)
157 with ops.name_scope("loop_body"):
158 loop_var = array_ops.placeholder(dtypes.int32, shape=[])
--> 159 loop_fn_outputs = loop_fn(loop_var)
160 new_ops = set(ops.get_default_graph().get_operations()) - existing_ops
161 iters = ops.convert_to_tensor(iters)

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/backprop.py in loop_fn(i)
1011 self._pop_tape()
1012 return self.gradient(y, flat_sources,
-> 1013 unconnected_gradients=unconnected_gradients)
1014
1015 try:

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/backprop.py in gradient(self, target, sources, output_gradients, unconnected_gradients)
944 flat_sources,
945 output_gradients=output_gradients,
--> 946 unconnected_gradients=unconnected_gradients)
947
948 if not self._persistent:

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/imperative_grad.py in imperative_grad(tape, target, sources, output_gradients, unconnected_gradients)
70 sources,
71 output_gradients,
---> 72 compat.as_str(unconnected_gradients.value))

/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/backprop.py in _gradient_function(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads)
129 return [None] * num_inputs
130
--> 131 return grad_fn(mock_op, *out_grads)
132
133

/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/array_grad.py in _ConcatGradV2(op, grad)
220 def _ConcatGradV2(op, grad):
221 return _ConcatGradHelper(
--> 222 op, grad, start_value_index=0, end_value_index=-1, dim_index=-1)
223
224

/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/array_grad.py in _ConcatGradHelper(op, grad, start_value_index, end_value_index, dim_index)
115 out_grads = array_ops.split(grad, sizes, non_neg_concat_dim)
116 else:
--> 117 if constant_op.is_constant(concat_dim):
118 # If concat_dim is a constant defined in a different context,
119 # then we duplicate it in the current context to avoid passing it

/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in is_constant(tensor_or_op)
293 def is_constant(tensor_or_op):
294 if isinstance(tensor_or_op, ops.Tensor):
--> 295 op = tensor_or_op.op
296 else:
297 op = tensor_or_op

/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in op(self)
925 def op(self):
926 raise AttributeError(
--> 927 "Tensor.op is meaningless when eager execution is enabled.")
928
929 @Property

AttributeError: Tensor.op is meaningless when eager execution is enabled.

Issue 2:
System information

Have I written custom code (as opposed to using a stock example script provided in TensorFlow): yes

OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Linux Ubuntu 16.04

TensorFlow installed from (source or binary): using nvidia containter: https://docs.nvidia.com/deeplearning/dgx/tensorflow-release-notes/rel_18.06.html#rel_18.06

TensorFlow version (use command below): 1.8.0

Python version: 3.5

Bazel version (if compiling from source): N/A

GCC/Compiler version (if compiling from source):N/A

CUDA/cuDNN version: 9.0.176

GPU model and memory: nvidia tesla v100

Exact command to reproduce:N/A

Describe the problem

I'm training an network using tf.estimator where I load a set of variables from an existing checkpoint. This is done using the warm start-up option from the estimator class. While the code works when running on a single GPU, It generates an error when I use the MirroredStrategy.

Source code / logs

The code:

ws = tf.estimator.WarmStartSettings(ckpt_to_initialize_from="init_checkpoint.ckpt"),
vars_to_warm_start="^((?!Logits).)*$"
)
self.trainer = tf.estimator.Estimator(
model_fn=self.get_model_fn, # First-class function
params=None, # HParams are passed using the config file
config=run_config,
warm_start_from=ws
)

This generate the following trace:

Traceback (most recent call last):
File "train.py", line 37, in <module>
experimenter.run_training_experiment(config)
File "/media/local/BDA_tf_framework/neuralnetwork/trainingexperimenter.py", line 39, in run_training_experiment
tf.estimator.train_and_evaluate(self.trainer, self.training_specs, self.eval_specs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 439, in train_and_evaluate
executor.run()
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 518, in run
self.run_local()
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 650, in run_local
hooks=train_hooks)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 363, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 841, in _train_model
return self._train_model_distributed(input_fn, hooks, saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 977, in _train_model_distributed
saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 986, in _train_with_estimator_spec
warm_starting_util.warm_start(*self._warm_start_settings)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/warm_starting_util.py", line 342, in warm_start
_warm_start_var(variable, ckpt_to_initialize_from, prev_var_name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/warm_starting_util.py", line 139, in _warm_start_var
"PartitionedVariable, but is {}".format(type(var)))
TypeError: var MUST be one of the following: a Variable, list of Variable or PartitionedVariable, but is <class 'tensorflow.contrib.distribute.python.values.MirroredVariable'>
Makefile:19: recipe for target 'train_gpu' failed
make: *** [train_gpu] Error 1