LSTM pertrained model execution error

username: krrishabh
cooked: <p>Need help… New in Torch</p>
<p>Wrote a code seeing here and there…</p>
<p>Doesn’t getting solved from 2 days… Please help it is emergency</p>
<pre><code class="lang-auto">#include &lt;torch/torch.h&gt;
#include &lt;torch/script.h&gt;
#include &lt;iostream&gt;
#include &lt;memory&gt;
#include &lt;dirent.h&gt;

using namespace std;

int main(int argc, const char* argv[]) {
  
  // Create the device we pass around based on whether CUDA is available.
  torch::Device device(torch::kCPU);
  if (torch::cuda::is_available()) {
    std::cout &lt;&lt; "CUDA is available! Training on GPU." &lt;&lt; std::endl;
    device = torch::Device(torch::kCUDA);
  }

  if (argc != 2) {
    std::cerr &lt;&lt; "usage: LMtesting &lt;path-to-exported-script-module&gt;\n";
    return -1;
  }

  torch::jit::script::Module module;
  try {
    // Deserialize the ScriptModule from a file using torch::jit::load().
    module = torch::jit::load(argv[1]);
  }
  catch (const c10::Error&amp; e) {
    std::cerr &lt;&lt; "error loading the model\n";
    return -1;
  }

  std::cout &lt;&lt; "ok\n";
  std::cout &lt;&lt; "Language Model\n\n";

  // Hyper parameters 

  const int64_t embed_size = 128;
  const int64_t hidden_size = 1024;
  const int64_t num_layers = 1;
  const int64_t num_samples = 1000;  // the number of words to be sampled
  const int64_t batch_size = 20;
  const int64_t sequence_length = 30;
  const size_t num_epochs = 5;
  const double learning_rate = 0.002;

  vector&lt;torch::jit::IValue&gt; inputs;
  vector&lt;float&gt; data(sequence_length*batch_size, 1.0);
  // Geting the error in next 3 lines
  torch::Tensor data_tensor = torch::from_blob(data.data(), {sequence_length,batch_size}.to(at::Device("cuda:0")); 
  torch::Tensor h0 = torch::from_blob(vector&lt;float&gt;(num_layers* batch_size * hidden_size, 0.0), {num_layers, batch_size, hidden_size}).to(at::Device("cuda:0"));
  torch::Tensor c0 = torch::from_blob(vector&lt;float&gt;(num_layers* batch_size * hidden_size, 0.0), {num_layers, batch_size, hidden_size}).to(at::Device("cuda:0"));
  
  inputs.push_back(data_tensor);
  inputs.push_back(h0);

  // Getting error in next 2 lines
  inputs.push(c0);
  torch::Tensor output = module-&gt;forward(inputs).toTensor().cpu();
  auto accessor = output.accessor&lt;float, 2&gt;();
}

</code></pre>

username: krrishabh
cooked: <p>Now It is properly compiled.</p>
<pre><code class="lang-auto">#include &lt;torch/torch.h&gt;
#include &lt;torch/script.h&gt;
#include &lt;iostream&gt;
#include &lt;memory&gt;
#include &lt;dirent.h&gt;

using namespace std;

int main(int argc, const char* argv[]) {
  
  // Create the device we pass around based on whether CUDA is available.
  torch::Device device(torch::kCPU);
  if (torch::cuda::is_available()) {
    std::cout &lt;&lt; "CUDA is available! Training on GPU." &lt;&lt; std::endl;
    device = torch::Device(torch::kCUDA);
  }

  if (argc != 2) {
    std::cerr &lt;&lt; "usage: LMtesting &lt;path-to-exported-script-module&gt;\n";
    return -1;
  }

  torch::jit::script::Module module;
  try {
    // Deserialize the ScriptModule from a file using torch::jit::load().
    module = torch::jit::load(argv[1]);
  }
  catch (const c10::Error&amp; e) {
    std::cerr &lt;&lt; "error loading the model\n";
    return -1;
  }

  std::cout &lt;&lt; "ok\n";
  std::cout &lt;&lt; "Language Model\n\n";

  // Hyper parameters
  int64_t embed_size = 128;
  int64_t hidden_size = 1024;
  int64_t num_layers = 1;
  int64_t num_samples = 1000;  // the number of words to be sampled
  int64_t batch_size = 20;
  int64_t sequence_length = 30;
  int64_t num_epochs = 5;
  double learning_rate = 0.002;


  
  torch::Tensor indata1 = torch::ones({sequence_length,batch_size}, torch::kLong);
  

  vector&lt;torch::jit::IValue&gt; inputs;
  vector&lt;float&gt; data(sequence_length*batch_size, 1.0);
  torch::Tensor data_tensor = torch::from_blob(data.data(), {sequence_length,batch_size});
  torch::Tensor h0 = torch::from_blob(std::vector&lt;float&gt;(num_layers* batch_size * hidden_size, 0.0).data(), {num_layers, batch_size, hidden_size});
  torch::Tensor c0 = torch::from_blob(std::vector&lt;float&gt;(num_layers* batch_size * hidden_size, 0.0).data(), {num_layers, batch_size, hidden_size});
  
  torch::jit::IValue tuple = torch::ivalue::Tuple::create({h0, c0});
  torch::Tensor output = module.forward({indata1, tuple}).toTensor().cpu();
  auto accessor = output.accessor&lt;float, 2&gt;();


}

</code></pre>
<p>But got error in execution…</p>
<pre><code class="lang-auto">terminate called after throwing an instance of 'std::runtime_error'
  what():  The following operation failed in the TorchScript interpreter.
Traceback of TorchScript, serialized code (most recent call last):
  File "code/newmodel.py", line 15, in forward
  bias = _5.bias
  hx, hx0, = argument_2
  input0 = torch.embedding(weight, input, -1, False, False)
           ~~~~~~~~~~~~~~~ &lt;--- HERE
  input1 = torch.dropout(input0, 0.20000000000000001, False)
  input2, _6, _7 = torch.lstm(input1, [hx, hx0], [_1, _2, _3, _4], True, 1, 0.20000000000000001, False, False, False)
</code></pre>
<p>My model.py is…</p>
<pre><code class="lang-auto">class RNNModel(nn.Module):
    """Container module with an encoder, a recurrent module, and a decoder."""

    def __init__(self, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False):
        super(RNNModel, self).__init__()
        self.drop = nn.Dropout(dropout)
        self.encoder = nn.Embedding(ntoken, ninp)
        if rnn_type in ['LSTM', 'GRU']:
            self.rnn = getattr(nn, rnn_type)(ninp, nhid, nlayers, dropout=dropout)
        else:
            try:
                nonlinearity = {'RNN_TANH': 'tanh', 'RNN_RELU': 'relu'}[rnn_type]
            except KeyError:
                raise ValueError( """An invalid option for `--model` was supplied,
                                 options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""")
            self.rnn = nn.RNN(ninp, nhid, nlayers, nonlinearity=nonlinearity, dropout=dropout)
        self.decoder = nn.Linear(nhid, ntoken)

        # Optionally tie weights as in:
        # "Using the Output Embedding to Improve Language Models" (Press &amp; Wolf 2016)
        # https://arxiv.org/abs/1608.05859
        # and
        # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
        # https://arxiv.org/abs/1611.01462
        if tie_weights:
            if nhid != ninp:
                raise ValueError('When using the tied flag, nhid must be equal to emsize')
            self.decoder.weight = self.encoder.weight

        self.init_weights()

        self.rnn_type = rnn_type
        self.nhid = nhid
        self.nlayers = nlayers

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, input, hidden):
        emb = self.drop(self.encoder(input))
        output, hidden = self.rnn(emb, hidden)
        output = self.drop(output)
        decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2)))
        return decoded.view(output.size(0), output.size(1), decoded.size(1)), hidden

    def init_hidden(self, bsz):
        weight = next(self.parameters())
        if self.rnn_type == 'LSTM':
            return (weight.new_zeros(self.nlayers, bsz, self.nhid),
                    weight.new_zeros(self.nlayers, bsz, self.nhid))
        else:
            return weight.new_zeros(self.nlayers, bsz, self.nhid)

</code></pre>
<p>Please Help me… <a class="mention" href="/u/tom">@tom</a>  (Sorry Sir to tag you)</p>

username: glaringlee
cooked: <p><a class="mention" href="/u/krrishabh">@krrishabh</a><br>
Hi, can you post this in jit category as well? I think the error happened in TorchScript interpreter, jit people probably can help.</p>

username: krrishabh
cooked: <p>Thank you sir I have solved this</p>

username: krrishabh
cooked: <p>Solution for the given Query (GRU) is</p>
<pre><code class="lang-auto">#include &lt;torch/torch.h&gt;
#include &lt;torch/script.h&gt;
#include &lt;iostream&gt;
#include &lt;memory&gt;
#include &lt;dirent.h&gt;

#include &lt;typeinfo&gt;

using namespace std;

int main(int argc, const char* argv[]) {
  
  // Create the device we pass around based on whether CUDA is available.
  torch::Device device(torch::kCPU);
  if (torch::cuda::is_available()) {
    std::cout &lt;&lt; "CUDA is available! Training on GPU." &lt;&lt; std::endl;
    device = torch::Device(torch::kCUDA);
  }

  if (argc != 2) {
    std::cerr &lt;&lt; "usage: LMtesting &lt;path-to-exported-script-module&gt;\n";
    return -1;
  }

  torch::jit::script::Module module;
  try {
    // Deserialize the ScriptModule from a file using torch::jit::load().
    module = torch::jit::load(argv[1]);
  }
  catch (const c10::Error&amp; e) {
    std::cerr &lt;&lt; "error loading the model\n";
    return -1;
  }

  std::cout &lt;&lt; "ok\n";
  std::cout &lt;&lt; "Language Model\n\n";

  // Hyper parameters
  int64_t embed_size = 100;
  int64_t hidden_size = 100;
  int64_t num_layers = 2;
  int64_t num_samples = 1000;  // the number of words to be sampled
  int64_t batch_size = 20;
  int64_t sequence_length = 35;
  int64_t num_epochs = 2;
  double learning_rate = 20;

  

  //module-&gt;eval();

  
  torch::Tensor indata1 = torch::zeros({sequence_length,batch_size}, torch::kLong);


  vector&lt;torch::jit::IValue&gt; inputs;
  vector&lt;float&gt; data(sequence_length*batch_size, 1.0);
  torch::Tensor data_tensor = torch::from_blob(data.data(), {sequence_length,batch_size}).cuda();
  torch::Tensor h0 = torch::from_blob(std::vector&lt;float&gt;(num_layers* batch_size * hidden_size, 0.0).data(), {num_layers, batch_size, hidden_size}).cuda();


 
  torch::Tensor output = module.forward({data_tensor, h0}).toTensor().cuda();
  
} 

</code></pre>

username: ivan1
cooked: <p><a class="mention" href="/u/krrishabh">@krrishabh</a>  In the last example, only h0 is inputted. How did you deal with torch::Tensor c0, before module.forward ?</p>