Expected Tensor but got GenericDict

huang624: <p><strong>Description:</strong><br>
I would like to infernce my model on LibTorch, but I got the error “Expected Tensor but got GenericDict”.</p>
<hr>
<p>Enviroment<br>
LibTorch: 1.13.1(maybe), but I downloag from pytorch with this link<br>
<a href="https://download.pytorch.org/libtorch/cu116/libtorch-shared-with-deps-1.13.1%2Bcu116.zip" class="onebox" target="_blank" rel="noopener nofollow ugc">https://download.pytorch.org/libtorch/cu116/libtorch-shared-with-deps-1.13.1%2Bcu116.zip</a><br>
Pytorch: 1.13.1</p>
<hr>
<p><strong>python code for exporting my model to use in c++:</strong></p>
<pre><code class="lang-auto">def add_targets(encodings, targets):
  encodings.update({'label': le.transform(targets)})

test_encodings = tokenizer(test_questions, truncation=True, padding=True)
add_targets(test_encodings, test_targets)

class Dataset(torch.utils.data.Dataset):
  def __init__(self, encodings):
    self.encodings = encodings
  def __getitem__(self, idx):
    return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
  def __len__(self):
    return len(self.encodings.input_ids)

test_dataset = Dataset(test_encodings)
</code></pre>
<p>Load model</p>
<pre><code class="lang-auto">from transformers import BertConfig, BertForSequenceClassification
config = BertConfig.from_pretrained(config_path)  
model = BertForSequenceClassification.from_pretrained(model_path, config=config)
</code></pre>
<p>Export</p>
<pre><code class="lang-auto">test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)

sample = next(iter(test_dataloader))
sample_input_id = sample['input_ids'].squeeze(1).to("cpu")
sample_mask = sample['attention_mask'].to("cpu")

traced_script_module = torch.jit.trace(model.cpu(), [sample_input_id, sample_mask], strict=False)
traced_script_module.save("traced_text_classification_model.pt")
</code></pre>
<p>model is available infernce on python</p>
<hr>
<p>C++ code for inference</p>
<pre><code class="lang-auto">#include &lt;iostream&gt;
#include &lt;map&gt;
#include &lt;fstream&gt;
#include &lt;string&gt;
#include &lt;sstream&gt;
#include &lt;cstring&gt;
#include &lt;vector&gt;
#include &lt;istream&gt;

#include &lt;torch/torch.h&gt;
#include &lt;torch/script.h&gt;

std::pair&lt;torch::Tensor, torch::Tensor&gt; preprocess(std::string text, std::map&lt;std::string, int&gt; token2id, int max_length, bool log = false){
    std::string pad_token = "[PAD]", start_token = "[CLS]", end_token = "[SEP]";
    int pad_token_id = token2id[pad_token], start_token_id = token2id[start_token], end_token_id = token2id[end_token]; //special token decode

    std::vector&lt;int&gt; input_ids(max_length, pad_token_id), masks(max_length, 0);  
    // init ele in input_ids  to pad token, and init attention mask to is 0
    input_ids[0] = start_token_id; masks[0] = 1; // start token -&gt; [CLS]

    std::string word;
    std::istringstream ss(text);
    
    int input_id = 1;
    while(getline(ss, word, ' ')) {
        int word_id = token2id[word];
        masks[input_id] = 1;
        input_ids[input_id++] = word_id;
        
        // if (log)
        std::cout &lt;&lt; word &lt;&lt; " : " &lt;&lt; word_id &lt;&lt; '\n';
    }
    

    masks[input_id] = 1;
    input_ids[input_id] = end_token_id;

    if (log){
        for (auto i : input_ids)
            std::cout &lt;&lt; i &lt;&lt; ' ';
        std::cout &lt;&lt; '\n';
    
        for (auto i : masks)
            std::cout &lt;&lt; i &lt;&lt; ' ';
        std::cout &lt;&lt; '\n';
    }

    auto input_ids_tensor = torch::tensor(input_ids).unsqueeze(0);
    auto masks_tensor = torch::tensor(masks).unsqueeze(0).unsqueeze(0);

    return std::make_pair(input_ids_tensor, masks_tensor);
}

struct Model{
    int max_length = 128;
    std::map&lt;std::string, int&gt; token2id;
    std::map&lt;int, std::string&gt; id2token;
    torch::jit::script::Module bert;

    void init_vocab(std::string vocab_path = "model/new_vocab.txt"){
        std::tie(token2id, id2token) = get_vocab(vocab_path);
    }

    void init_bert(std::string bert_path = "/user_data/web/model/traced_text_classification_model.pt"){
        bert = load_model(bert_path);
    }

    std::pair&lt;std::map&lt;std::string, int&gt;, std::map&lt;int, std::string&gt;&gt; get_vocab(std::string vocab_path){
        std::map&lt;std::string, int&gt; token2id;
        std::map&lt;int, std::string&gt; id2token;

        std::fstream newfile;
        newfile.open(vocab_path, std::ios::in);

        std::string line;
        while(getline(newfile, line)){
            char *token = strtok(const_cast&lt;char*&gt;(line.c_str()), " ");
            char *token_id = strtok(nullptr, " ");

            token2id[token] = std::stoi(token_id);
            id2token[std::stoi(token_id)] = token;
        }
        newfile.close();

        return std::make_pair(token2id, id2token);
    }

    torch::jit::script::Module load_model(std::string  model_path){
        torch::jit::script::Module module;
        try {
            module = torch::jit::load(model_path);
        }
        catch (const c10::Error&amp; e) {
            std::cerr &lt;&lt; "error loading the model\n";
        }
        return module;
    }
};

int main(int argc, char** argv){
    std::cout &lt;&lt; "start" &lt;&lt; std::endl;
    
    // set seed
    int seed = 42;
    torch::manual_seed(seed);
    torch::cuda::manual_seed(seed);

    auto model = Model();
    model.init_vocab();
    model.init_bert();

    auto token2id = model.token2id;

    std::string text = "今 天 天 氣 真 好";
    std::cout &lt;&lt; text &lt;&lt; std::endl;

    torch::Tensor input_ids, masks;
    std::tie(input_ids, masks) = preprocess(text, token2id, model.max_length);
    std::cout &lt;&lt; input_ids &lt;&lt; std::endl;
    std::cout &lt;&lt; masks &lt;&lt; std::endl;

    std::vector&lt;torch::jit::IValue&gt; inputs;
    inputs.push_back(input_ids);
    inputs.push_back(masks);

    auto outputs = model.bert.forward(inputs).toTensor();
    std::string pred = std::to_string(int(outputs.argmax().item&lt;int&gt;()));
    std::cout &lt;&lt; "Prediction: " &lt;&lt; pred &lt;&lt; std::endl;
    return 0;
}
</code></pre>
<p>The issue is here<br>
<strong>auto outputs = model.bert.forward(inputs).toTensor();</strong></p>
<p>Error<br>
terminate called after throwing an instance of ‘c10::Error’<br>
what():  Expected Tensor but got GenericDict<br>
Exception raised from reportToTensorTypeError at …/aten/src/ATen/core/ivalue.cpp:942 (most recent call first):<br>
frame <span class="hashtag">#0:</span> c10::Error::Error(c10::SourceLocation, std::string) + 0x57 (0x7f8639757dd7 in /user_data/libtorch/lib/libc10.so)<br>
frame <span class="hashtag">#1:</span> c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&amp;) + 0x64 (0x7f8639721c1c in /user_data/libtorch/lib/libc10.so)<br>
frame <span class="hashtag">#2:</span> c10::IValue::reportToTensorTypeError() const + 0x58 (0x7f8620c70fb8 in /user_data/libtorch/lib/libtorch_cpu.so)<br>
frame <span class="hashtag">#3:</span> c10::IValue::toTensor() &amp;&amp; + 0x4b (0x5566fba76df7 in ./web)<br>
frame <span class="hashtag">#4:</span> main + 0x4c9 (0x5566fba721fd in ./web)<br>
frame <span class="hashtag">#5:</span> __libc_start_main + 0xf3 (0x7f861ed46083 in /lib/x86_64-linux-gnu/libc.so.6)<br>
frame <span class="hashtag">#6:</span> _start + 0x2e (0x5566fba70d6e in ./web)</p>
<hr>
<p>similar issues:</p>
<aside class="onebox githubissue" data-onebox-src="https://github.com/triton-inference-server/server/issues/3348">
  <header class="source">

      <a href="https://github.com/triton-inference-server/server/issues/3348" target="_blank" rel="noopener nofollow ugc">github.com/triton-inference-server/server</a>
  </header>

  <article class="onebox-body">
    <div class="github-row">
  <div class="github-icon-container" title="Issue">
	  <svg width="60" height="60" class="github-icon" viewbox="0 0 14 16" aria-hidden="true"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg>
  </div>

  <div class="github-info-container">
    <h4>
      <a href="https://github.com/triton-inference-server/server/issues/3348" target="_blank" rel="noopener nofollow ugc">PyTorch execute failure: isTensor(); Expected Tensor but got GenericList</a>
    </h4>

    <div class="github-info">
      <div class="date">
        opened <span class="discourse-local-date" data-format="ll" data-date="2021-09-10" data-time="02:50:09" data-timezone="UTC">02:50AM - 10 Sep 21 UTC</span>
      </div>

        <div class="date">
          closed <span class="discourse-local-date" data-format="ll" data-date="2021-09-10" data-time="17:11:36" data-timezone="UTC">05:11PM - 10 Sep 21 UTC</span>
        </div>

      <div class="user">
        <a href="https://github.com/rexlow" target="_blank" rel="noopener nofollow ugc">
          <img alt="rexlow" src="https://avatars.githubusercontent.com/u/7680796?v=4" class="onebox-avatar-inline" width="20" height="20">
          rexlow
        </a>
      </div>
    </div>

    <div class="labels">
    </div>
  </div>
</div>

  <div class="github-row">
    <p class="github-body-container">**Description**
I've built a custom Yolov5 model and traced it via a [export sc<span class="show-more-container"><a href="" rel="noopener" class="show-more">…</a></span><span class="excerpt hidden">ript](https://github.com/ultralytics/yolov5/blob/master/export.py) from UltraLytics. Testing it via their [inference script](https://github.com/ultralytics/yolov5/blob/master/detect.py) and my Python native code works. Now I attempt to deploy it with Triton.

While deploying it Triton did not throw any error, it is when making a client call only I get this error.

A similar error can be found in #2594 , not sure why is this closed?? Perhaps @CoderHam can have a look at this issue? I'm passing a batched numpy array not tensor.

```
PyTorch execute failure: isTensor() INTERNAL ASSERT FAILED at "/tmp/tritonbuild/pytorch/build/include/torch/ATen/core/ivalue_inl.h":157, please report a bug to PyTorch. Expected Tensor but got GenericList
Exception raised from toTensor at /tmp/tritonbuild/pytorch/build/include/torch/ATen/core/ivalue_inl.h:157 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string&lt;char, std::char_traits&lt;char&gt;, std::allocator&lt;char&gt; &gt;) + 0x6c (0x7f56f28405cc in /opt/tritonserver/backends/pytorch/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string&lt;char, std::char_traits&lt;char&gt;, std::allocator&lt;char&gt; &gt; const&amp;) + 0xfa (0x7f56f2806d4e in /opt/tritonserver/backends/pytorch/libc10.so)
frame #2: &lt;unknown function&gt; + 0xf558 (0x7f56f2d9a558 in /opt/tritonserver/backends/pytorch/libtriton_pytorch.so)
frame #3: &lt;unknown function&gt; + 0x155a2 (0x7f56f2da05a2 in /opt/tritonserver/backends/pytorch/libtriton_pytorch.so)
frame #4: TRITONBACKEND_ModelInstanceExecute + 0x411 (0x7f56f2da1a91 in /opt/tritonserver/backends/pytorch/libtriton_pytorch.so)
frame #5: &lt;unknown function&gt; + 0x2f2fd7 (0x7f573ca89fd7 in /opt/tritonserver/bin/../lib/libtritonserver.so)
frame #6: &lt;unknown function&gt; + 0xfdfe0 (0x7f573c894fe0 in /opt/tritonserver/bin/../lib/libtritonserver.so)
frame #7: &lt;unknown function&gt; + 0xd6d84 (0x7f573c2d1d84 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
frame #8: &lt;unknown function&gt; + 0x9609 (0x7f573c76c609 in /usr/lib/x86_64-linux-gnu/libpthread.so.0)
frame #9: clone + 0x43 (0x7f573bfbf293 in /usr/lib/x86_64-linux-gnu/libc.so.6)
```

**Triton Information**
Triton docker version 21.03

**Config file**
```
name: "my_model"
platform: "pytorch_libtorch"
max_batch_size : 128
input {
    name: "input__0"
    data_type: TYPE_FP32
    format: FORMAT_NCHW
    dims: [ 3, 640, 640 ]
}
output {
    name: "output__0"
    data_type: TYPE_FP32
    dims: [ 20 ]
    label_filename: "labels.txt"
}
instance_group {
  count: 1
  kind: KIND_GPU
  gpus: 0
}
dynamic_batching {
  preferred_batch_size: [ 1, 2, 4, 8, 16 ]
  max_queue_delay_microseconds: 100
}
```

**Python Client**

```
image = cv2.resize(img, (640, 640), interpolation = cv2.INTER_CUBIC)
image = image.astype(np.float32)
image /= 255. # [640, 640, 3]
image = np.transpose(image, (2, 0, 1)) # [3, 640, 640]
data = np.expand_dims(image, 0) # [1, 3, 640, 640]

input_1 = tchttp.InferInput("input__0", (1, 3, 640, 640), "FP32")
input_1.set_data_from_numpy(data)
outputs = [tchttp.InferRequestedOutput(name) for name in ["output__0"]]
response = triton_client.infer(
  mmodel_name='my_model', 
  inputs=[input_1],
  outputs=outputs,
  model_version='1')
logits = response.as_numpy('output__0')
```</span></p>
  </div>

  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>

<p>and they solve the issue using changing there forward() function in model.<br>
Well, I am confused that how I change my function, changing before exporting in python, or changing in c++.</p>
<p>Do everyone have suggestion or help for how I can fix my code.</p>
<hr>
<p>Thanks for your help!</p>