Why do 2 identical Neural Network fowarding a different result

username: Kallinteris-Andreas
cooked: <p>my torch::module::nn:</p>
<pre><code class="lang-auto">struct Net : torch::nn::Module {
	Net(int numIn, int numOut, int numHid, const size_t hid_count=1) {
		assert(hid_count &gt;= 1);
		first = register_parameter("inputW", torch::rand({numIn, numHid}))/numHid;
		middle = new torch::Tensor[hid_count-1];
		for (int i = 1; i != hid_count; i++)
			middle[i] = register_parameter("hidW"+std::to_string(i), torch::rand({numHid, numHid}))/numHid;
		last = register_parameter("outputW", torch::rand({numHid, numOut}))/numOut;
		h_c = hid_count;
		n_h = numHid;
	}
	torch::Tensor forward(torch::Tensor input) {
		torch::Tensor output_layer,h;
		h = (torch::mm(input, first));
		for (int i = 1; i != h_c; i++)
			h = torch::sigmoid(torch::mm(h, middle[i]));
		output_layer = (torch::mm(h, last));
		return output_layer;
	}
	torch::Tensor first, last, *middle;
	size_t h_c, n_h;
};
</code></pre>
<p>my unit test:</p>
<pre><code class="lang-auto">{
		const int input_nodes = 20, output_nodes=10, hidden_nodes = 2, hidden_count = 1;
		Net nn0 (input_nodes, output_nodes, hidden_nodes, hidden_count);
		Net nn1 (input_nodes, output_nodes, hidden_nodes, hidden_count);
		for (size_t i = 0; i &lt; nn0.parameters().size(); i++ ){//copy all parameters
			torch::Tensor t = nn0.parameters()[i].detach().clone();
			nn1.parameters()[i].set_data(t);
			//nn1.parameters()[i] = nn0.parameters()[i].detach().clone();
		}
		for (size_t i = 0; i &lt; nn0.parameters().size(); i++ )//assert that all the parameters are equal
			assert(torch::sum(nn1.parameters()[i] == nn0.parameters()[i]).item&lt;float&gt;() == nn0.parameters()[i].numel());
		const torch::Tensor input = torch::ones({1,input_nodes});
		std::cout &lt;&lt; nn0.forward(input) - nn1.forward(input) &lt;&lt; std::endl;
		assert(torch::sum(nn0.forward(input) == nn1.forward(input)).item&lt;float&gt;() == (float) output_nodes);
	}
</code></pre>
<p>Unit test explanation:<br>
First we create 2 nn::modules then we copy all the parameters(Tensors)<br>
Second we assert that all the parameters are copied succefully<br>
Third run both nn::modules (which have equal parameters) with the same input and observe different results</p>
<p>Sample output (of the delta of the nn fowards):</p>
<pre><code class="lang-auto"> 0.3192  0.2746 -0.0047  0.2926 -0.4053 -0.1266 -0.2181 -0.4602  0.8585  0.0089
</code></pre>
<p>Also in this unit test we can see that on the same NN if we give the same inputs, it will give the same output</p>
<pre><code class="lang-auto">	{//assert that fowarding the same NN with the same inputs provides the same output
		const int input_nodes = 20, output_nodes=10, hidden_nodes = 2, hidden_count = 1;
		Net nn0 (input_nodes, output_nodes, hidden_nodes, hidden_count);
		const torch::Tensor input = torch::rand({1,input_nodes});
		assert(torch::sum(nn0.forward(input) == nn0.forward(input)).item&lt;float&gt;() == (float) output_nodes);
	}
</code></pre>

username: Kallinteris-Andreas
cooked: <p>Update:<br>
The source of the issue appers to be that torch::nn::module::parameters() does reference the same underline torch::Tensor as it’s research value</p>
<pre><code class="lang-auto">struct Net : torch::nn::Module {
	Net(int numIn, int numOut, int numHid, const size_t hid_count=1) {
                ...
		first = register_parameter("inputW", torch::rand({numIn, numHid}))/numHid;
                ...
        }
        ...
}
</code></pre>
<pre><code class="lang-auto">{//in some unit test
...
		assert(torch::sum(nn1.parameters()[0] == nn1.first).item&lt;float&gt;() == nn1.parameters()[0].numel());
...
}//this fails
</code></pre>
<p>Any suggestion on how to fix this?</p>

username: Matej_Kompanek
cooked: <p>Your problem is caused by dividing the result of ‘register_parameter’ → it creates different tensor which is then assigned to member variable (thus can’t be affected from outside the module by editing ‘.parameters()’)<br>
the part → “/numHid”:</p>
<pre><code class="lang-auto">first = register_parameter("inputW", torch::rand({numIn, numHid}))/numHid;
</code></pre>
<p>Full example using approach from: <a href="https://discuss.pytorch.org/t/how-to-copy-network-parameters-in-libtorch-c-api/32221">How to copy parameters</a>:</p>
<pre><code class="lang-auto">struct NetImpl : torch::nn::Module {
	NetImpl(int numIn, int numOut, int numHid, const size_t hid_count = 1) {
		assert(hid_count &gt;= 1);
		first = register_parameter("inputW", torch::rand({ numIn, numHid }));
		middle = new torch::Tensor[hid_count - 1];
		for (int i = 1; i != hid_count; i++)
			middle[i] = register_parameter("hidW" + std::to_string(i), torch::rand({ numHid, numHid }));
		last = register_parameter("outputW", torch::rand({ numHid, numOut }));
		h_c = hid_count;
		n_h = numHid;
	}
	torch::Tensor forward(torch::Tensor input)
	{
		torch::Tensor output_layer, h;
		h = (torch::mm(input, first));
		for (int i = 1; i != h_c; i++)
			h = torch::sigmoid(torch::mm(h, middle[i]));
		output_layer = (torch::mm(h, last));
		return output_layer;
	}
	torch::Tensor first, last, *middle;
	size_t h_c, n_h;
};

TORCH_MODULE(Net);

void main(int argc, char** argv)
{
	try
	{
		const int input_nodes = 20, output_nodes = 10, hidden_nodes = 2, hidden_count = 1;

		Net nn0(input_nodes, output_nodes, hidden_nodes, hidden_count);
		Net nn1(input_nodes, output_nodes, hidden_nodes, hidden_count);

		{
			torch::autograd::GradMode::set_enabled(false);
			auto src = nn0-&gt;named_parameters(true /*recurse*/);
			auto dst = nn1-&gt;named_parameters(true /*recurse*/);

			for (auto&amp; val : src)
			{
				auto name = val.key();
				auto* t = dst.find(name);
				if (t != nullptr)
				{
					t-&gt;copy_(val.value());
				}
			}

			torch::autograd::GradMode::set_enabled(true);
		}

		const torch::Tensor input = torch::ones({ 1,input_nodes });

		std::cout &lt;&lt; "Diff: " &lt;&lt; nn0-&gt;forward(input) - nn1-&gt;forward(input) &lt;&lt; std::endl;
	}
	catch (const c10::Error&amp; e)
	{
		std::cout &lt;&lt; e.what() &lt;&lt; std::endl;
	}
	catch (const std::runtime_error&amp; e)
	{
		std::cout &lt;&lt; e.what() &lt;&lt; std::endl;
	}
	
	system("PAUSE");
}
</code></pre>