Error while Testing Multithreaded Solution

username: QFuser
cooked: <p>I know you usually use processes in Pytroch but I wanted to see if you can do it with multi threading at least it reduce overhead which is nice when you work in a limited environment. So I jumped in …</p>
<p>I made it so it starts running but soon it cause a bug somewhere a debug Abort() is called deep in the system. I know something is wrong with my batch.input</p>
<p>void ThreadHandler::train(MyNet extern_net, int seed, Options options)<br>
{<br>
std::vector&lt;int64_t&gt; expected_size = { 29, 1, 4 };<br>
auto train_set = ClimateDataset(data).map(torch::data::transforms::Stack&lt;&gt;());<br>
torch::manual_seed(seed);<br>
extern_net-&gt;to(options.device);</p>
<pre><code>torch::optim::Adam optimizer(extern_net-&gt;parameters(), torch::optim::AdamOptions(options.learning_rate));

auto train_loader = torch::data::make_data_loader&lt;torch::data::samplers::RandomSampler&gt;(
    std::move(train_set),
    torch::data::DataLoaderOptions().batch_size(options.train_batch_size));


for (auto&amp; batch : *train_loader) {
	
    optimizer.zero_grad();
    
    
    auto inputs = batch.data;//.to(options.device);
    auto targets = batch.target.to(options.device);;
    std::vector&lt;int64_t&gt; input_size = inputs.sizes().vec();
    std::vector&lt;int64_t&gt; target_size = targets.sizes().vec();
    //for debugging and condinal breaks only
    int64_t ist = input_size.operator[](2);
    int64_t tes = target_size.operator[](2);
    //AT_ASSERT(input_size == expected_size);
    torch::Tensor prediction = extern_net-&gt;forward(inputs);

	auto loss = torch::l1_loss(prediction, targets);
	//AT_ASSERT(!std::isnan(loss.template item&lt;float&gt;()));
	loss.backward();
	optimizer.step();
}
average(extern_net);
//std::cout &lt;&lt; "ThreadHandler: " &lt;&lt; std::this_thread::get_id() &lt;&lt; " is done" &lt;&lt; std::endl;
stop();
</code></pre>
<p>}<br>
if .to(options.device) is called it triggers it but if not its end here</p>
<p><span class="hashtag-raw">#pragma</span> once<br>
<span class="hashtag-raw">#include</span> <br>
<span class="hashtag-raw">#include</span> &lt;torch/torch.h&gt;</p>
<p>struct MyNetImpl : torch::nn::Module {<br>
MyNetImpl(int64_t input_size, int64_t hidden_size, int64_t hidden_size2, int64_t hidden_size3, int64_t output_size) :<br>
i2h(register_module(“i2h”, torch::nn::Linear(input_size, hidden_size))),<br>
h2h(register_module(“h2h”, torch::nn::Linear(hidden_size, hidden_size2))),<br>
h2h2(register_module(“h2h2”, torch::nn::Linear(hidden_size2, hidden_size3))),<br>
h2h3(register_module(“h2h3”, torch::nn::Linear(hidden_size3, hidden_size3))),<br>
h2o(register_module(“h2o”, torch::nn::Linear(hidden_size3, output_size))) {}</p>
<pre><code>torch::Tensor forward(torch::Tensor x1) {
    //  torch::Tensor xi = torch::cat({ x1, x2, x3 }, 1);
   //  std::cout &lt;&lt; "Input tensor x1: " &lt;&lt; x1 &lt;&lt; std::endl;
    torch::Tensor xm = torch::relu(i2h-&gt;forward(x1));
    //std::cout &lt;&lt; "Input tensor xm: " &lt;&lt; xm &lt;&lt; std::endl;
    torch::Tensor xm2 = torch::relu(h2h-&gt;forward(xm));
    torch::Tensor xm3 = torch::relu(h2h2-&gt;forward(xm2));
    torch::Tensor xo = torch::relu(h2h3-&gt;forward(xm3));
    //std::cout &lt;&lt; "Input tensor xo: " &lt;&lt; xo &lt;&lt; std::endl;
    torch::Tensor x = h2o-&gt;forward(xo);
    // std::cout &lt;&lt; "Input tensor x: " &lt;&lt; x &lt;&lt; std::endl;

    return x;
}


torch::nn::Linear i2h, h2h, h2h2, h2h3, h2o;
</code></pre>
<p>};<br>
TORCH_MODULE(MyNet);</p>
<p>Basically the first line of forwards I’m sure it has something to do with the multi threading. I’m bit on a lose here  and yes the dimensions seems fine so I’m not sure what I should check.</p>
<p>Stacktrace:</p>
<div class="md-table">
<table>
<thead>
<tr>
<th></th>
<th>ucrtbased.dll!00007ff8d4cc2e65()</th>
<th>Unknown</th>
</tr>
</thead>
<tbody>
<tr>
<td></td>
<td>ucrtbased.dll!00007ff8d4cc3003()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>ucrtbased.dll!00007ff8d4cdab0d()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>ucrtbased.dll!00007ff8d4cd9a90()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>vcruntime140_1d.dll!00007ff9a757223a()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>vcruntime140_1d.dll!00007ff9a7572ec5()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>vcruntime140_1d.dll!00007ff9a7572f57()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>vcruntime140_1d.dll!00007ff9a7576dbb()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>ntdll.dll!00007ff9c36523af()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>ntdll.dll!00007ff9c36014b4()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>ntdll.dll!00007ff9c3650ebe()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>KernelBase.dll!00007ff9c101cf19()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>vcruntime140d.dll!00007ff937f2b760()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>c10.dll!00007ff8d599809d()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff85fef1cdc()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cuda.dll!00007ff896eef11b()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cuda.dll!00007ff896a9a3de()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cuda.dll!00007ff896c1b26c()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cuda.dll!00007ff896c9f722()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861098d41()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861015ec5()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861183d1c()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff86122ada9()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff86186b7a5()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff864d37d5a()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff864bb7184()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff864bb64fa()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff864c3a570()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff864c5fc8f()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861098d41()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861015ec5()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861015c3d()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8612070ec()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff86186b6d6()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff85f9c7100()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8603661f1()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff860353225()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff862546800()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff86257e63c()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8625d9842()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861098d41()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861015ec5()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861015c3d()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8612070ec()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff861a83e96()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff85f9c6790()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8674aadc6()</td>
<td>Unknown</td>
</tr>
<tr>
<td></td>
<td>torch_cpu.dll!00007ff8674f3583()</td>
<td>Unknown</td>
</tr>
<tr>
<td>&gt;</td>
<td>DisTest.exe!MyNetImpl::forward(at::Tensor x1) Line 16</td>
<td>C++</td>
</tr>
<tr>
<td></td>
<td>DisTest.exe!ThreadHandler::train(MyNet extern_net, int seed, Options options) Line 65</td>
<td>C++</td>
</tr>
<tr>
<td></td>
<td>DisTest.exe!ThreadHandler::start() Line 15</td>
<td>C++</td>
</tr>
<tr>
<td></td>
<td>[External Code]</td>
<td></td>
</tr>
</tbody>
</table>
</div>

username: QFuser
cooked: <p>Okay my bad this happened because I accessed the same instance of MyNet from diffrent threads it seems.(but I have still errors)</p>