Why is my numpy implementation 8x faster than pytorch forward pass?

username: craftyvisage
cooked: <p>The output from the code below is:</p>
<pre><code class="lang-auto">pytorch answer = [1.14435797]
pytorch takes 12.410856008529663 s
my answer = [1.14435797]
my imlementation takes 1.566734790802002 s
</code></pre>
<p>So why is the pytorch forward pass 8x slower than numpy here?</p>
<pre><code class="lang-auto">import IPython as ipy
import numpy as np
import torch
import time

class Net(torch.nn.Module):
    def __init__(self, input_size, layer_width):
        super(Net, self).__init__()
        self.input_size = input_size
        output_size = 1
        self.fc1 = torch.nn.Linear(input_size, layer_width)
        self.fc2 = torch.nn.Linear(layer_width, layer_width)
        self.fc3 = torch.nn.Linear(layer_width, output_size)

    def forward(self, x):
        x = torch.nn.functional.relu(self.fc1(x))
        x = torch.nn.functional.relu(self.fc2(x))
        x = torch.exp(self.fc3(x))
        return x

    def get_input_size(self):
        return self.input_size

def main():
    torch.set_default_dtype(torch.float64)
    torch.set_grad_enabled(False)
    input_size = 20
    layer_width = 20
    net = Net(input_size, layer_width)

    N_evals = 100000

    pytorch_x = torch.ones(input_size)
    pytorch_answer = net(pytorch_x).detach().numpy()
    print("pytorch answer = " + str(pytorch_answer))
    t0 = time.time()
    pytorch_x = torch.ones(input_size)
    for i in range(N_evals):
        pytorch_answer = net(pytorch_x)
    t1 = time.time()
    print("pytorch takes " + str(t1 - t0) + " s")

    x = pytorch_x.detach().numpy()
    w1 = net.fc1.weight.detach().numpy()
    w2 = net.fc2.weight.detach().numpy()
    w3 = net.fc3.weight.detach().numpy()
    b1 = net.fc1.bias.detach().numpy()
    b2 = net.fc2.bias.detach().numpy()
    b3 = net.fc3.bias.detach().numpy()

    y = np.matmul(w1, x) + b1
    y = y * (y &gt; 0)
    y = np.matmul(w2, y) + b2
    y = y * (y &gt; 0)
    my_answer = np.exp(np.matmul(w3, y) + b3)
    print("my answer = " + str(my_answer))
    
    t0 = time.time()
    x = np.ones(input_size)
    for i in range(N_evals):
        y = np.matmul(w1, x) + b1
        y = y * (y &gt; 0)
        y = np.matmul(w2, y) + b2
        y = y * (y &gt; 0)
        my_answer = np.exp(np.matmul(w3, y) + b3)
    t1 = time.time()
    print("my imlementation takes " + str(t1 - t0) + " s")

if __name__ == "__main__":
    main()
</code></pre>

username: Oli
cooked: <p>Don’t know, but it would be fun to see which part takes time. Can you run this profiler on it? <a href="https://jiffyclub.github.io/snakeviz/" rel="nofollow noopener">snakeviz</a></p>

username: craftyvisage
cooked: <p>I ran the profiler as you suggested. First I made the code easier to profile by splitting the pytorch and my numpy runs. See below.</p>
<p>Not sure what exactly you expected. Nothing jumps out to me other than everything looks slow. The times 13.7 and 8.44 don’t add up to 14.1 s. Not sure what to make of that.</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/2X/2/25d7d3fbbd2c736ced7655acff207af074e46ef7.png" data-download-href="https://discuss.pytorch.org/uploads/default/25d7d3fbbd2c736ced7655acff207af074e46ef7" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/2X/2/25d7d3fbbd2c736ced7655acff207af074e46ef7_2_690x250.png" alt="image" data-base62-sha1="5oM4HJjjEyldHYfCQp3QIpzzgEf" width="690" height="250" srcset="https://discuss.pytorch.org/uploads/default/optimized/2X/2/25d7d3fbbd2c736ced7655acff207af074e46ef7_2_690x250.png, https://discuss.pytorch.org/uploads/default/optimized/2X/2/25d7d3fbbd2c736ced7655acff207af074e46ef7_2_1035x375.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/2X/2/25d7d3fbbd2c736ced7655acff207af074e46ef7_2_1380x500.png 2x" data-dominant-color="BFC0C3"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1440×523 30.2 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<pre><code class="lang-auto">import IPython as ipy
import numpy as np
import torch
import time

class Net(torch.nn.Module):
    def __init__(self, input_size, layer_width):
        super(Net, self).__init__()
        self.input_size = input_size
        output_size = 1
        self.fc1 = torch.nn.Linear(input_size, layer_width)
        self.fc2 = torch.nn.Linear(layer_width, layer_width)
        self.fc3 = torch.nn.Linear(layer_width, output_size)

    def forward(self, x):
        x = torch.nn.functional.relu(self.fc1(x))
        x = torch.nn.functional.relu(self.fc2(x))
        x = torch.exp(self.fc3(x))
        return x

    def get_input_size(self):
        return self.input_size

def pytorch_run(net, input_size, N_evals):
    t0 = time.time()
    pytorch_x = torch.ones(input_size)
    for i in range(N_evals):
        pytorch_answer = net(pytorch_x)
    t1 = time.time()
    return t1 - t0

def my_run(w1, w2, w3, b1, b2, b3, input_size, N_evals):
    t0 = time.time()
    x = np.ones(input_size)
    for i in range(N_evals):
        y = np.matmul(w1, x) + b1
        y = y * (y &gt; 0)
        y = np.matmul(w2, y) + b2
        y = y * (y &gt; 0)
        my_answer = np.exp(np.matmul(w3, y) + b3)
    t1 = time.time()
    return t1 - t0

def main():
    torch.set_default_dtype(torch.float32)
    torch.set_grad_enabled(False)
    input_size = 20
    layer_width = 20
    net = Net(input_size, layer_width)

    N_evals = 100000

    pytorch_x = torch.ones(input_size)
    pytorch_answer = net(pytorch_x).detach().numpy()
    print("pytorch answer = " + str(pytorch_answer))
    pytorch_time = pytorch_run(net, input_size, N_evals)
    print("pytorch takes " + str(pytorch_time) + " s")

    x = pytorch_x.detach().numpy()
    w1 = net.fc1.weight.detach().numpy()
    w2 = net.fc2.weight.detach().numpy()
    w3 = net.fc3.weight.detach().numpy()
    b1 = net.fc1.bias.detach().numpy()
    b2 = net.fc2.bias.detach().numpy()
    b3 = net.fc3.bias.detach().numpy()

    x = np.ones(input_size)
    y = np.matmul(w1, x) + b1
    y = y * (y &gt; 0)
    y = np.matmul(w2, y) + b2
    y = y * (y &gt; 0)
    my_answer = np.exp(np.matmul(w3, y) + b3)
    print("my answer = " + str(my_answer))

    my_time = my_run(w1, w2, w3, b1, b2, b3, input_size, N_evals)
    print("my imlementation takes " + str(my_time) + " s")


if __name__ == "__main__":
    main()
</code></pre>

username: ptrblck
cooked: <p>You might see some overhead for tiny workloads in PyTorch.<br>
Changing the <code>input_size</code> and <code>layer_width</code> to <code>1000</code> gives:</p>
<pre><code class="lang-python">pytorch answer = [1.0642374]
pytorch takes 16.20141328498721 s
my answer = [1.06423738]
my imlementation takes 150.2599041541107 s
</code></pre>
<p>I had to kill the run after a few minutes using <code>2000</code> for both variables, as numpy took too long.</p>