Lstm expects two hidden states

username: bokonon
cooked: <p>I am using the <code>forward</code> method below. The code compiles successfully, but I get an error stating that <code>lstm expects two hidden states</code>.</p>
<p>Any guidance would be appreciated.</p>
<pre><code class="lang-auto">#include &lt;ATen/ATen.h&gt;
#include "rnn.h"

std::vector&lt;torch::Tensor&gt; PackedLSTMImpl::flat_weights() const {
  // Organize all weights in a flat vector in the order
  // (weight_ih_l0, weight_hh_l0, bias_ih_l0, bias_hh_l0)
  // repeated for each layer (next to each other).
  std::vector&lt;torch::Tensor&gt; flat;
  std::cout &lt;&lt; "Flattening weights..." &lt;&lt; std::endl;

  const auto num_directions = rnn_-&gt;options.bidirectional() ? 2 : 1;
  for (int64_t layer = 0; layer &lt; rnn_-&gt;options.num_layers(); layer++) {
    for (auto direction = 0; direction &lt; num_directions; direction++) {
       const auto layer_idx =
           static_cast&lt;size_t&gt;((layer * num_directions) + direction);
      string layer_num = std::to_string(layer);
      flat.push_back(rnn_-&gt;named_parameters()["weight_ih_l" + layer_num][layer_idx]);
      flat.push_back(rnn_-&gt;named_parameters()["weight_hh_l" + layer_num][layer_idx]);
      if (rnn_-&gt;options.bias()) {
        flat.push_back(rnn_-&gt;named_parameters()["bias_ih_l" + layer_num][layer_idx]);
        flat.push_back(rnn_-&gt;named_parameters()["bias_hh_l" + layer_num][layer_idx]);
      }
      // flat.push_back(rnn_-&gt;named_parameters()["weight_ih_l0"][layer_idx]);
      // flat.push_back(rnn_-&gt;named_parameters()["weight_hh_l0"][layer_idx]);
      // if (rnn_-&gt;options.bias()) {
      //   flat.push_back(rnn_-&gt;named_parameters()["bias_ih_l0"][layer_idx]);
      //   flat.push_back(rnn_-&gt;named_parameters()["bias_hh_l0"][layer_idx]);
      // }
    }
  }
  return flat;
}

std::tuple&lt;torch::Tensor, torch::Tensor&gt; PackedLSTMImpl::forward(const torch::Tensor&amp; input,
                                             const at::Tensor&amp; lengths,
                                             torch::Tensor state) {
  std::cout &lt;&lt; "in PackedLSTMImpl::forward()..." &lt;&lt; std::endl;
  if (!state.defined()) {
    // 2 for hidden state and cell state, then #layers, batch size, state size
    // const auto batch_size = input.size(rnn_-&gt;options.batch_first_ ? 0 : 1);
    const auto max_batch_size = lengths[0].item().toLong();
    const auto num_directions = rnn_-&gt;options.bidirectional() ? 2 : 1;
    state = torch::zeros({2, rnn_-&gt;options.num_layers() * num_directions,
                          max_batch_size, rnn_-&gt;options.hidden_size()},
                         input.options());
  }
  torch::Tensor output, hidden_state, cell_state;

  std::vector&lt;torch::Tensor&gt; tensor_vector = flat_weights();

  torch::TensorList flattened_weights[tensor_vector.size()];
  for (int i=1; i&lt;tensor_vector.size(); i++)
    flattened_weights[i] = tensor_vector[i];

  // aten::lstm.data(Tensor data, Tensor batch_sizes, Tensor[] hx, Tensor[] params, bool has_biases, int num_layers, float dropout, bool train, bool bidirectional) -&gt; (Tensor, Tensor, Tensor)
  // at::lstm(at::Tensor const&amp;, c10::ArrayRef&lt;at::Tensor&gt;, c10::ArrayRef&lt;at::Tensor&gt;, bool, long, double, bool, bool, bool)
  torch::TensorList hidden_states = {state[0], state[1]};
  std::tie(output, hidden_state, cell_state) = at::lstm(
      input, lengths, hidden_states, flattened_weights,
      rnn_-&gt;options.bias(), rnn_-&gt;options.num_layers(), rnn_-&gt;options.dropout(),
      rnn_-&gt;is_training(), rnn_-&gt;options.bidirectional());
 
  return {output, torch::stack({hidden_state, cell_state})};
}
</code></pre>

username: ptrblck
cooked: <p>Could you try to use <code>std::make_tuple(hidden_state, cell_state)</code> instead of stacking these tensors?</p>

username: bokonon
cooked: <p>If I change <code>return {output, torch::stack({hidden_state, cell_state})};</code> to <code>return {output, std::make_tuple(hidden_state, cell_state)};</code>, then I receive the following compilation error:</p>
<pre><code class="lang-auto">In member function ‘std::tuple&lt;at::Tensor, at::Tensor&gt; PackedLSTMImpl::forward(const at::Tensor&amp;, const at::Tensor&amp;, at::Tensor)’:
/home/ec2-user/hands-on-ml-cplus-plus/Chapter11/pytorch/rnn.cc:80:60: error: could not convert ‘{output, std::make_tuple(_Elements&amp;&amp; ...) [with _Elements = {at::Tensor&amp;, at::Tensor&amp;}](cell_state)}’ from ‘&lt;brace-enclosed initializer list&gt;’ to ‘std::tuple&lt;at::Tensor, at::Tensor&gt;’
   return {output, std::make_tuple(hidden_state, cell_state)};
</code></pre>

username: bokonon
cooked: <p>Interestingly enough, if you use the <code>at::lstm</code> with different parameters, as shown below, you will get this error message at runtime: <code>Dimension out of range (expected to be in range of [-1, 0], but got 2)</code></p>
<pre><code class="lang-auto">#include &lt;ATen/ATen.h&gt;
#include "rnn.h"

std::vector&lt;torch::Tensor&gt; PackedLSTMImpl::flat_weights() const {
  // Organize all weights in a flat vector in the order
  // (weight_ih_l0, weight_hh_l0, bias_ih_l0, bias_hh_l0)
  // repeated for each layer (next to each other).
  std::vector&lt;torch::Tensor&gt; flat;
  std::cout &lt;&lt; "Flattening weights..." &lt;&lt; std::endl;

  const auto num_directions = rnn_-&gt;options.bidirectional() ? 2 : 1;
  for (int64_t layer = 0; layer &lt; rnn_-&gt;options.num_layers(); layer++) {
    for (auto direction = 0; direction &lt; num_directions; direction++) {
       const auto layer_idx =
           static_cast&lt;size_t&gt;((layer * num_directions) + direction);
      string layer_num = std::to_string(layer);
      flat.push_back(rnn_-&gt;named_parameters()["weight_ih_l" + layer_num][layer_idx]);
      flat.push_back(rnn_-&gt;named_parameters()["weight_hh_l" + layer_num][layer_idx]);
      if (rnn_-&gt;options.bias()) {
        flat.push_back(rnn_-&gt;named_parameters()["bias_ih_l" + layer_num][layer_idx]);
        flat.push_back(rnn_-&gt;named_parameters()["bias_hh_l" + layer_num][layer_idx]);
      }
      // flat.push_back(rnn_-&gt;named_parameters()["weight_ih_l0"][layer_idx]);
      // flat.push_back(rnn_-&gt;named_parameters()["weight_hh_l0"][layer_idx]);
      // if (rnn_-&gt;options.bias()) {
      //   flat.push_back(rnn_-&gt;named_parameters()["bias_ih_l0"][layer_idx]);
      //   flat.push_back(rnn_-&gt;named_parameters()["bias_hh_l0"][layer_idx]);
      // }
    }
  }
  return flat;
}

std::tuple&lt;torch::Tensor, torch::Tensor&gt; PackedLSTMImpl::forward(const torch::Tensor&amp; input,
                                             const at::Tensor&amp; lengths,
                                             torch::Tensor state) {
  std::cout &lt;&lt; "in PackedLSTMImpl::forward()..." &lt;&lt; std::endl;
  if (!state.defined()) {
    // 2 for hidden state and cell state, then #layers, batch size, state size
    // const auto batch_size = input.size(rnn_-&gt;options.batch_first_ ? 0 : 1);
    const auto max_batch_size = lengths[0].item().toLong();
    const auto num_directions = rnn_-&gt;options.bidirectional() ? 2 : 1;
    state = torch::zeros({2, rnn_-&gt;options.num_layers() * num_directions,
                          max_batch_size, rnn_-&gt;options.hidden_size()},
                         input.options());
  }
  torch::Tensor output, hidden_state, cell_state;

  // aten::lstm.data(Tensor data, Tensor batch_sizes, Tensor[] hx, Tensor[] params, bool has_biases, int num_layers, float dropout, bool train, bool bidirectional) -&gt; (Tensor, Tensor, Tensor)
  // at::lstm(at::Tensor const&amp;, c10::ArrayRef&lt;at::Tensor&gt;, c10::ArrayRef&lt;at::Tensor&gt;, bool, long, double, bool, bool, bool)
  at::TensorList hidden_states = {state[0], state[1]};
  std::tie(output, hidden_state, cell_state) = at::lstm(
      input, hidden_states, (at::TensorList)flat_weights(),
      // input, lengths, hidden_states, flattened_weights,
      rnn_-&gt;options.bias(), rnn_-&gt;options.num_layers(), rnn_-&gt;options.dropout(),
      rnn_-&gt;is_training(), rnn_-&gt;options.bidirectional(), rnn_-&gt;options.batch_first());
  

  return {output, torch::stack({hidden_state, cell_state})};
}
</code></pre>