Issue 1:
System information

I have written custom code.

Linux Ubuntu 19.04, Debian 9.9

TensorFlow installed from binary

TensorFlow version 2.0.0-b1, 1.14, 1.15.0

Python version: 3.7.1, 3.7.3

CUDA/cuDNN version: 10.0, 10.1

GPU model and memory: Tesla T4, Tesla P4, P100, does not exist on k80

Describe the current behavior

The result on the GPU is significantly different from CPU. It is correct on CPU, but wrong on GPU. The difference on GPU is much bigger than zero.

Describe the expected behavior

The print out of the difference should be zero.

Code to reproduce the issue

tf 2.0.0-b1

import tensorflow as tf
import numpy as np

@tf.function
def sample_y_nn_tf(w, X_sample, config):

layers = config['layers']
current_w_index = 0
h = X_sample
w_shape = tf.shape(input=w)

l = layers[0]
w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
h = tf.einsum('lm,ikjmn->likjn', h, w_current) + b_current
current_w_index = current_w_index + (l[0] + 1) * l[1]

for l in layers[1:]:
h = tf.nn.elu(h)
w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
h = tf.einsum('likjm,ikjmn->likjn',h,w_current)+b_current
current_w_index = current_w_index+(l[0]+1)*l[1]

h = tf.squeeze(h, axis=4)

return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

print('permutation', permutation)

with tf.device("/cpu:0"):
means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
print('difference on cpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))

with tf.device("/gpu:0"):
means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
print('difference on gpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))

Output:

difference on cpu 0.0

difference on gpu 37953784.0

tf.1.14

from tensorflow.python.framework.versions import VERSION
import tensorflow as tf
import numpy as np
import os

print(tf.__version__)

@tf.function
def sample_y_nn_tf(w, X_sample, config):

layers = config['layers']
current_w_index = 0
h = X_sample
w_shape = tf.shape(input=w)

l = layers[0]
w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
h = tf.einsum('lm,ikjmn->likjn', h, w_current) + b_current
current_w_index = current_w_index + (l[0] + 1) * l[1]

for l in layers[1:]:
h = tf.nn.elu(h)
w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
h = tf.einsum('likjm,ikjmn->likjn',h,w_current)+b_current
current_w_index = current_w_index+(l[0]+1)*l[1]

h = tf.squeeze(h, axis=4)

return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
precision = np.exp(np.random.normal(size=(config['batch_size'],64,512))).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

conf = tf.ConfigProto()
conf.gpu_options.allow_growth = True
sess = tf.Session(config=conf)

print('permutation', sess.run(permutation))

with tf.device("/cpu:0"):
means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
net_cpu = tf.gather(means,permutation, axis=1)-means_p

with tf.device("/device:GPU:0"):
means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
net_gpu = tf.gather(means,permutation, axis=1)-means_p

net_cpu_result = sess.run(net_cpu)
net_gpu_result = sess.run(net_gpu)
print("tensorflow version: {}".format(VERSION))
print('difference on cpu', np.sum(np.abs(net_cpu_result)))
print('difference on gpu', np.sum(np.abs(net_gpu_result)))

Output:

difference on cpu 0.0

difference on gpu 39042744.0

Other info / logs

I did not check the tf2 version on K80, P100.

The bug probably persists when using Pythonic slicing operations.

Issue 2:
System information

Have I written custom code (as opposed to using a stock example script provided in TensorFlow): Yes

OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Linux Ubuntu 18.04

Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device:

TensorFlow installed from (source or binary): binary

TensorFlow version (use command below): v1.14.0-rc1-22-gaf24dc91b5 1.14.0

Python version: 3.6.7

GCC/Compiler version (if compiling from source): 8.2.0

CUDA/cuDNN version: cuda-10.0

GPU model and memory:

Describe the current behavior

tf.matmul on tensorflow-gpu gave wrong results. Here is a simplified version of the code.

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

r = [[ 1.0, 0.0], [0.0, 1.0]]

x, y = np.meshgrid(list(range(400)), list(range(400)))
coords = np.stack([x,y],-1).reshape((400,400,2,1))
coords = tf.convert_to_tensor(coords,dtype=tf.float32)

r1 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords,feed_dict={r1:r})

plt.matshow(ret[:,:,0,0])
plt.show()

When I ran it on my tensorflow-gpu, here is the result:

Looks like it stops computing halfway through and gave the rest 0 as a result.

Describe the expected behavior

Here is the result with CPU:

Below is my old post. Initially, I thought the problem was related to TFRecord, but seems like this problem occurs without even using tfrecord too. For completeness, I keep the old example code with tfrecord.

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

def parser(serialized_example):
fs = tf.io.parse_single_example(
serialized_example,
features={ "r": tf.FixedLenFeature([4], tf.float32) })
fs["r"] = tf.reshape(fs["r"], [2, 2])
return fs

r = [[ 1.0, 0.0],[0.0, 1.0]]

with tf.io.TFRecordWriter("cc.test") as tfrecord_writer:
feature = {"r": tf.train.Feature(float_list=tf.train.FloatList(value=np.array(r).flatten() ))}
example = tf.train.Example(features=tf.train.Features(feature=feature))
tfrecord_writer.write(example.SerializeToString())
dataset = tf.data.TFRecordDataset(["cc.test"])
dataset = dataset.map(parser).repeat().make_one_shot_iterator()
features = dataset.get_next()

x, y = tf.meshgrid(list(range(400)), list(range(400)))
coords = tf.stack([x, y], -1) #(h,w,2)
coords = tf.expand_dims(tf.cast(coords,tf.float32),-1) #(h,w,2,1)

r1 = features["r"]
r2 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords[:,:,0,0])
plt.matshow(ret)
plt.show()

The code will create "cc.test" file to save the variable r and load it as r1. Then matmul r1 with some big varibles.