Issue 1:
When I run https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/dist_test/python/mnist_replica.py, the distributed example

I can run it with CPU(1 ps and 2 workers all on CPU in same machine )

but when I run it with GPU, I set the GPU

number to 2(1 ps on CPU and 2 workers on GPU in same machine), the first worker runs normally, but when I start the second one, it shows the following errors

Traceback (most recent call last):
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 972, in _do_call
return fn(*args)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 954, in _run_fn
status, run_metadata)
File "/home/mlabs/anaconda3/lib/python3.5/contextlib.py", line 66, in __exit__
next(self.gen)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/errors.py", line 463, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors.InvalidArgumentError: WhereOp: Race condition between counting the number of true elements and writing them. When counting, saw 1363 elements; but when writing their indices, saw 12 elements.
[[Node: report_uninitialized_variables/boolean_mask/Where = Where[_device="/job:worker/replica:0/task:1/cpu:0"](report_uninitialized_variables/boolean_mask/Reshape_1)]]
[[Node: report_uninitialized_variables/boolean_mask/Where_G11 = _Recv[client_terminated=false, recv_device="/job:worker/replica:0/task:1/gpu:1", send_device="/job:worker/replica:0/task:1/cpu:0", send_device_incarnation=-7465067838139069765, tensor_name="edge_29_report_uninitialized_variables/boolean_mask/Where", tensor_type=DT_INT64, _device="/job:worker/replica:0/task:1/gpu:1"]()]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "mnist_replica.py", line 266, in <module>
tf.app.run()
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "mnist_replica.py", line 223, in main
config=sess_config)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/supervisor.py", line 722, in prepare_or_wait_for_session
max_wait_secs=max_wait_secs)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/session_manager.py", line 349, in wait_for_session
is_ready, not_ready_msg = self._model_ready(sess)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/session_manager.py", line 435, in _model_ready
return self._ready(self._ready_op, sess, "Model not ready")
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/session_manager.py", line 404, in _ready
ready_value = sess.run(op)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 717, in run
run_metadata_ptr)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 915, in _run
feed_dict_string, options, run_metadata)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 965, in _do_run
target_list, options, run_metadata)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 985, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors.InvalidArgumentError: WhereOp: Race condition between counting the number of true elements and writing them. When counting, saw 1363 elements; but when writing their indices, saw 12 elements.
[[Node: report_uninitialized_variables/boolean_mask/Where = Where[_device="/job:worker/replica:0/task:1/cpu:0"](report_uninitialized_variables/boolean_mask/Reshape_1)]]
[[Node: report_uninitialized_variables/boolean_mask/Where_G11 = _Recv[client_terminated=false, recv_device="/job:worker/replica:0/task:1/gpu:1", send_device="/job:worker/replica:0/task:1/cpu:0", send_device_incarnation=-7465067838139069765, tensor_name="edge_29_report_uninitialized_variables/boolean_mask/Where", tensor_type=DT_INT64, _device="/job:worker/replica:0/task:1/gpu:1"]()]]

Caused by op 'report_uninitialized_variables/boolean_mask/Where', defined at:
File "mnist_replica.py", line 266, in <module>
tf.app.run()
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "mnist_replica.py", line 201, in main
global_step=global_step)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/supervisor.py", line 310, in __init__
ready_op=ready_op, ready_for_local_init_op=ready_for_local_init_op)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/training/supervisor.py", line 399, in _init_ready_op
ready_op = variables.report_uninitialized_variables()
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/ops/variables.py", line 1167, in report_uninitialized_variables
return array_ops.boolean_mask(variable_names_tensor, variables_mask)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 950, in boolean_mask
return _apply_mask_1d(tensor, mask)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 925, in _apply_mask_1d
indices = squeeze(where(mask), squeeze_dims=[1])
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3072, in where
result = _op_def_lib.apply_op("Where", input=input, name=name)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 749, in apply_op
op_def=op_def)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 2380, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/mlabs/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1298, in __init__
self._traceback = _extract_stack()

InvalidArgumentError (see above for traceback): WhereOp: Race condition between counting the number of true elements and writing them. When counting, saw 1363 elements; but when writing their indices, saw 12 elements.
[[Node: report_uninitialized_variables/boolean_mask/Where = Where[_device="/job:worker/replica:0/task:1/cpu:0"](report_uninitialized_variables/boolean_mask/Reshape_1)]]
[[Node: report_uninitialized_variables/boolean_mask/Where_G11 = _Recv[client_terminated=false, recv_device="/job:worker/replica:0/task:1/gpu:1", send_device="/job:worker/replica:0/task:1/cpu:0", send_device_incarnation=-7465067838139069765, tensor_name="edge_29_report_uninitialized_variables/boolean_mask/Where", tensor_type=DT_INT64, _device="/job:worker/replica:0/task:1/gpu:1"]()]]

Issue 2:
I have the same problem as issue #4033 and tensorflow/models#486 happening to me in my own project with the nightly wheel. It is running fine on CPU, the problem only happens on GPU (titan x pascal).

The code that's causing the problem is this:

pair_idxs = tf.where(tf.greater_equal(iou, params.thresh))

(iou is a tensor, params.thresh is a python float)

WIth the error message:

InvalidArgumentError (see above for traceback): WhereOp: Race condition between counting the number of true elements and writing them. When counting, saw 31641 elements; but when writing their indices, saw 187 elements.
[[Node: Where = Where[_device="/job:localhost/replica:0/task:0/cpu:0"](GreaterEqual/_25)]]

Cuda:

-rw-r--r-- 1 foo bar 52M Dec 8 20:06 libcublas_device.a
lrwxrwxrwx 1 foo bar 16 Dec 8 20:06 libcublas.so -> libcublas.so.8.0
lrwxrwxrwx 1 foo bar 19 Dec 8 20:07 libcublas.so.8.0 -> libcublas.so.8.0.45
-rwxr-xr-x 1 foo bar 40M Dec 8 20:06 libcublas.so.8.0.45
-rw-r--r-- 1 foo bar 46M Dec 8 20:07 libcublas_static.a
-rw-r--r-- 1 foo bar 546K Dec 8 20:07 libcudadevrt.a
lrwxrwxrwx 1 foo bar 16 Dec 8 20:06 libcudart.so -> libcudart.so.8.0
lrwxrwxrwx 1 foo bar 19 Dec 8 20:07 libcudart.so.8.0 -> libcudart.so.8.0.44
-rwxr-xr-x 1 foo bar 406K Dec 8 20:06 libcudart.so.8.0.44
-rw-r--r-- 1 foo bar 757K Dec 8 20:07 libcudart_static.a
lrwxrwxrwx 1 foo bar 15 Dec 8 20:06 libcufft.so -> libcufft.so.8.0
lrwxrwxrwx 1 foo bar 18 Dec 8 20:07 libcufft.so.8.0 -> libcufft.so.8.0.44
-rwxr-xr-x 1 foo bar 140M Dec 8 20:06 libcufft.so.8.0.44
-rw-r--r-- 1 foo bar 124M Dec 8 20:07 libcufft_static.a
lrwxrwxrwx 1 foo bar 16 Dec 8 20:06 libcufftw.so -> libcufftw.so.8.0
lrwxrwxrwx 1 foo bar 19 Dec 8 20:07 libcufftw.so.8.0 -> libcufftw.so.8.0.44
-rwxr-xr-x 1 foo bar 466K Dec 8 20:06 libcufftw.so.8.0.44
-rw-r--r-- 1 foo bar 42K Dec 8 20:07 libcufftw_static.a
lrwxrwxrwx 1 foo bar 17 Dec 8 20:06 libcuinj64.so -> libcuinj64.so.8.0
lrwxrwxrwx 1 foo bar 20 Dec 8 20:06 libcuinj64.so.8.0 -> libcuinj64.so.8.0.44
-rwxr-xr-x 1 foo bar 6.2M Dec 8 20:06 libcuinj64.so.8.0.44
-rw-r--r-- 1 foo bar 1.6M Dec 8 20:07 libculibos.a
lrwxrwxrwx 1 foo bar 16 Dec 8 20:06 libcurand.so -> libcurand.so.8.0
lrwxrwxrwx 1 foo bar 19 Dec 8 20:07 libcurand.so.8.0 -> libcurand.so.8.0.44
-rwxr-xr-x 1 foo bar 57M Dec 8 20:07 libcurand.so.8.0.44
-rw-r--r-- 1 foo bar 57M Dec 8 20:06 libcurand_static.a
lrwxrwxrwx 1 foo bar 18 Dec 8 20:06 libcusolver.so -> libcusolver.so.8.0
lrwxrwxrwx 1 foo bar 21 Dec 8 20:07 libcusolver.so.8.0 -> libcusolver.so.8.0.44
-rwxr-xr-x 1 foo bar 52M Dec 8 20:06 libcusolver.so.8.0.44
-rw-r--r-- 1 foo bar 22M Dec 8 20:07 libcusolver_static.a
lrwxrwxrwx 1 foo bar 18 Dec 8 20:07 libcusparse.so -> libcusparse.so.8.0
lrwxrwxrwx 1 foo bar 21 Dec 8 20:06 libcusparse.so.8.0 -> libcusparse.so.8.0.44
-rwxr-xr-x 1 foo bar 41M Dec 8 20:07 libcusparse.so.8.0.44
-rw-r--r-- 1 foo bar 50M Dec 8 20:06 libcusparse_static.a

cuDNN:

lrwxrwxrwx 1 foo bar 13 Jul 27 07:55 libcudnn.so -> libcudnn.so.5
lrwxrwxrwx 1 foo bar 17 Jul 27 07:55 libcudnn.so.5 -> libcudnn.so.5.1.5
-rwxr-xr-x 1 foo bar 76M Jul 27 07:53 libcudnn.so.5.1.5
-rw-r--r-- 1 foo bar 67M Jul 27 07:53 libcudnn_static.a

tenserflow.__version__: 0.12.head