Convert to onnx after quantize_dynamic or optimize_for_mobile - both crash ? any workaround?

username: cdahms
cooked: <p>I need to make a saved model much smaller than it is currently (will be running on an embedded device with very limited memory), preferably down to 1/3 or 1/4 of the size.</p>
<p>Also, due to the limited memory situation, I have to convert to onnx so I can inference without PyTorch (PyTorch won’t fit).  Of course I can train on a desktop without such limitations.</p>
<p>I’m doing something for the company I work for so I can’t share the exact network, but here is full code to produce a simplified version for illustrative purposes so I can ask my question:</p>
<pre><code class="lang-auto"># MNIST.py

# Net Layout:
# batchSize x 1 x 28 x 28
#     conv1 Conv2d(1, 6, 5)
# batchSize x 6 x 24 x 24
#     relu(x)
#     max_pool2d(x, kernel_size=2)
# batchSize x 6 x 12 x 12
#     conv2 Conv2d(6, 16, 5)
# batchSize x 16 x 8 x 8
#     relu(x)
#     max_pool2d(x, kernel_size=2)
# batchSize x 16 x 4 x 4
#     view(-1, 16 * 4 * 4)    Note: 16 * 4 * 4 = 256
# batchSize x 1 x 256
#     fc1 Linear(256, 120)
#     relu(x)
# batchSize x 1 x 120
#     fc2 Linear(120, 84)
#     relu(x)
# batchSize x 1 x 84
#     fc3 Linear(84, 10)
# batchSize x 1 x 10

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader

import torchvision

import cv2
import numpy as np
import random
from termcolor import colored

IMAGE_WIDTH = 28
IMAGE_HEIGHT = 28

TRANSFORM = torchvision.transforms.Compose([torchvision.transforms.Resize((IMAGE_WIDTH, IMAGE_HEIGHT)),
                                            torchvision.transforms.ToTensor(),
                                            torchvision.transforms.Normalize([0.5], [0.5])
                                            ])
BATCH_SIZE = 64
NUM_EPOCHS = 5

class MnistNet(nn.Module):

    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(256, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
    # end function

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), kernel_size=2)
        x = F.max_pool2d(F.relu(self.conv2(x)), kernel_size=2)
        x = x.view(-1, 256)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
    # end function

# end class

def main():
    trainDataset = torchvision.datasets.MNIST('built_in_dataset', train=True, download=True, transform=TRANSFORM)

    # choose a 20% subset of the train idxs to save time
    trainDatasetIdxs = list(np.arange(len(trainDataset)))
    trainDatasetIdxs = random.sample(trainDatasetIdxs, k=round(len(trainDataset) * 0.2))
    trainDataset = torch.utils.data.Subset(trainDataset, trainDatasetIdxs)

    # # randomly pick out an image to show, if desired
    # randTrainIdx = random.randint(0, len(trainDataset) - 1)
    # ptTrainImage, trainLabelIdx = trainDataset[randTrainIdx]
    # pilTrainImage = torchvision.transforms.ToPILImage()(ptTrainImage)
    # openCvTrainImage = np.array(pilTrainImage)
    # # would convert RGB to BGR here if image was color
    # print('random training image trainLabelIdx = ' + str(trainLabelIdx))
    # cv2.imshow('image', openCvTrainImage)
    # cv2.waitKey()

    trainDataLoader = DataLoader(trainDataset, batch_size=BATCH_SIZE, shuffle=True)

    # declare net, loss function, and optimizer
    mnistNet = MnistNet()
    lossFunction = nn.CrossEntropyLoss()
    optimizer = optim.Adam(mnistNet.parameters())

    # get device (cuda or cpu)
    if torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        print(colored('WARNING: CUDA does not seem to be available, using CPU', 'yellow'))
        device = torch.device('cpu')
    # end if

    # set network to device
    mnistNet.to(device)

    # set network to train mode
    mnistNet.train()

    print('beginning training . . .')

    # for each epoch . . .
    epoch = 1
    while epoch &lt; NUM_EPOCHS + 1:

        # variables to calculate loss and accuracy within the epoch
        epochLosses = []
        epochAccuracies = []

        # for each batch . . .
        for i, element in enumerate(trainDataLoader):
            # break out the input images and ground truths, note these are Tensors
            inputImages, gndTrths = element

            inputImages = inputImages.to(device)
            gndTrths = gndTrths.to(device)

            # clear gradients from the previous step
            optimizer.zero_grad()

            # get net output
            outputs = mnistNet(inputImages)
            # calculate loss
            loss = lossFunction(outputs, gndTrths)
            # call backward() to compute gradients
            loss.backward()
            # update parameters using gradients
            optimizer.step()

            # append the current classification loss to the list of epoch losses
            epochLosses.append(loss.item())

            # calculate current classification accuracy

            # get the highest scoring classification for each prediction
            _, predictions = torch.max(outputs.data, 1)

            # number of gndTrths and predictions should always be the same, log an error if this is not the case
            if gndTrths.size(0) != predictions.size(0):
                print(colored('ERROR: gndTrths.size(0) != predictions.size(0)', 'red'))
            # end if

            # determine the number of correct predictions for the current batch
            correctPredictions = 0
            for j in range(len(gndTrths)):
                if predictions[j].item() == gndTrths[j].item():
                    correctPredictions += 1
                # end if
            # end for

            # append the current batch accuracy to the list of accuracies
            epochAccuracies.append(correctPredictions / gndTrths.size(0))
        # end for

        # calculate epoch loss and accuracy from the respective lists
        epochLoss = sum(epochLosses) / len(epochLosses)
        epochAccuracy = sum(epochAccuracies) / len(epochAccuracies)

        print('epoch ' + str(epoch) + ', epochLoss = ' + '{:.4f}'.format(epochLoss) + ', epochAccuracy = ' + '{:.4f}'.format(epochAccuracy * 100) + '%')

        epoch += 1
    # end for

    # save the model as a PyTorch graph
    torch.save(mnistNet.state_dict(), 'MNIST.pt')

    # save the model as an ONNX graph
    dummyInput = torch.randn(BATCH_SIZE, 1, IMAGE_WIDTH, IMAGE_HEIGHT).to(device)
    torch.onnx.export(mnistNet, dummyInput, 'MNIST.onnx')
</code></pre>
<p>This works great and <code>MNIST.onnx</code> can be inferenced as expected.</p>
<p>Now for the <code>quantize_dynamic</code> attempt.  If I change the end (after the big training for loop) to:</p>
<pre><code class="lang-auto">    torch.save(mnistNet.state_dict(), 'MNIST.pt')

    # quantize
    mnistNet.to('cpu')
    mnistNet = torch.quantization.quantize_dynamic(mnistNet, {torch.nn.Linear}, dtype=torch.qint8)

    torch.save(mnistNet.state_dict(), 'MNISTquant.pt')

    # save the model as an ONNX graph
    dummyInput = torch.randn(BATCH_SIZE, 1, IMAGE_WIDTH, IMAGE_HEIGHT).to('cpu')
    dummyOutput = mnistNet(dummyInput)
    torch.onnx.export(mnistNet, dummyInput, 'MNIST.onnx', verbose=True,
                      operator_export_type=torch.onnx.OperatorExportTypes.ONNX, example_outputs=dummyOutput)
</code></pre>
<p>I get:</p>
<pre><code class="lang-auto">$ python3 MNIST4.py 
beginning training . . .
epoch 1, epochLoss = 0.7125, epochAccuracy = 77.9422%
epoch 2, epochLoss = 0.1899, epochAccuracy = 94.0409%
epoch 3, epochLoss = 0.1199, epochAccuracy = 96.1686%
epoch 4, epochLoss = 0.0898, epochAccuracy = 97.1410%
epoch 5, epochLoss = 0.0682, epochAccuracy = 97.8391%
Traceback (most recent call last):
  File "MNIST4.py", line 243, in &lt;module&gt;
    main()
  File "MNIST4.py", line 186, in main
    torch.onnx.export(mnistNet, dummyInput, 'MNIST.onnx', verbose=True,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/__init__.py", line 271, in export
    return utils.export(model, args, f, export_params, verbose, training,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 88, in export
    _export(model, args, f, export_params, verbose, training, input_names, output_names,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 691, in _export
    _model_to_graph(model, args, verbose, input_names,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 454, in _model_to_graph
    graph, params, torch_out, module = _create_jit_graph(model, args,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 417, in _create_jit_graph
    graph, torch_out = _trace_and_get_graph_from_model(model, args)
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 374, in _trace_and_get_graph_from_model
    orig_state_dict_keys = _unique_state_dict(model).keys()
  File "/usr/local/lib/python3.8/dist-packages/torch/jit/_trace.py", line 69, in _unique_state_dict
    filtered_dict[k] = v.detach()
AttributeError: 'torch.dtype' object has no attribute 'detach'
</code></pre>
<p>Line 186 is the <code>torch.onnx.export</code> line.</p>
<p>As an alternative, I figured I’d try <code>optimize_for_mobile</code>, which requires using <code>torch.jit.trace</code> to convert to a <code>ScriptModule</code>.  Here is my attempt at that (as before, changing only the end after the big training for loop):</p>
<pre><code class="lang-auto">    # save the model as a PyTorch graph
    torch.save(mnistNet.state_dict(), 'MNIST.pt')

    mnistNet.eval()
    mnistNet = mnistNet.to('cpu')

    dummyInput = torch.randn(BATCH_SIZE, 1, IMAGE_WIDTH, IMAGE_HEIGHT).to('cpu')
    mnistNet = torch.jit.trace(mnistNet, dummyInput)

    mnistNet =  torch.utils.mobile_optimizer.optimize_for_mobile(mnistNet)

    dummyOutput = mnistNet(dummyInput)

    torch.onnx.export(mnistNet, dummyInput, 'MNIST.onnx', verbose=True,
                      operator_export_type=torch.onnx.OperatorExportTypes.ONNX, example_outputs=dummyOutput)
</code></pre>
<p>Which produces:</p>
<pre><code class="lang-auto">$ python3 MNIST5.py 
beginning training . . .
epoch 1, epochLoss = 0.6987, epochAccuracy = 78.6154%
epoch 2, epochLoss = 0.1879, epochAccuracy = 94.2154%
epoch 3, epochLoss = 0.1295, epochAccuracy = 95.8610%
epoch 4, epochLoss = 0.0984, epochAccuracy = 96.8418%
epoch 5, epochLoss = 0.0807, epochAccuracy = 97.5233%
/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:889: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:930.)
  result = self.forward(*input, **kwargs)
Traceback (most recent call last):
  File "MNIST5.py", line 244, in &lt;module&gt;
    main()
  File "MNIST5.py", line 189, in main
    torch.onnx.export(mnistNet, dummyInput, 'MNIST.onnx', verbose=True,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/__init__.py", line 271, in export
    return utils.export(model, args, f, export_params, verbose, training,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 88, in export
    _export(model, args, f, export_params, verbose, training, input_names, output_names,
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 676, in _export
    with select_model_mode_for_export(model, training):
  File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__
    return next(self.gen)
  File "/usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py", line 38, in select_model_mode_for_export
    is_originally_training = model.training
  File "/usr/local/lib/python3.8/dist-packages/torch/jit/_script.py", line 561, in __getattr__
    return super(RecursiveScriptModule, self).__getattr__(attr)
  File "/usr/local/lib/python3.8/dist-packages/torch/jit/_script.py", line 291, in __getattr__
    return super(ScriptModule, self).__getattr__(attr)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 947, in __getattr__
    raise AttributeError("'{}' object has no attribute '{}'".format(
AttributeError: 'RecursiveScriptModule' object has no attribute 'training'
</code></pre>
<p>I did see these posts:</p><aside class="quote quote-modified" data-post="1" data-topic="62634">
  <div class="title">
    <div class="quote-controls"></div>
    <img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/liron_mor_yosef/48/12953_2.png" class="avatar">
    <a href="https://discuss.pytorch.org/t/can-i-convert-torch-script-module-to-nn-module/62634">Can i convert torch script module to nn module</a> <a class="badge-category__wrapper " href="/c/jit/13"><span data-category-id="13" data-drop-close="true" class="badge-category " title="A category for TorchScript and the PyTorch JIT compiler"><span class="badge-category__name">jit</span></span></a>
  </div>
  <blockquote>
    Hi, 
i have a problem when converting torch script module in to ONNX module, 
but it’s works just fine on “normal” module, 
i open a bug request about that at <a href="https://github.com/pytorch/pytorch/issues/30512" rel="noopener nofollow ugc">https://github.com/pytorch/pytorch/issues/30512</a>, 
but if i could convert the torch script module to nn module it’s will solve my issue, 
Thanks, 
Liron 
code example:(you can ignore the test_normal ) 
    def _test_normal(self, num_classes, dummy_input):
        model = torchvision.models.resnet18(num_classes=num_classes)
        model_sta…
  </blockquote>
</aside>
<aside class="onebox githubissue" data-onebox-src="https://github.com/pytorch/pytorch/issues/30512">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/issues/30512" target="_blank" rel="noopener nofollow ugc">github.com/pytorch/pytorch</a>
  </header>

  <article class="onebox-body">
    <div class="github-row">
  <div class="github-icon-container" title="Issue">
	  <svg width="60" height="60" class="github-icon" viewBox="0 0 14 16" aria-hidden="true"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg>
  </div>

  <div class="github-info-container">
    <h4>
      <a href="https://github.com/pytorch/pytorch/issues/30512" target="_blank" rel="noopener nofollow ugc">failed to convert  torch.jit.ScriptModule to ONNX (crash)</a>
    </h4>

    <div class="github-info">
      <div class="date">
        opened <span class="discourse-local-date" data-format="ll" data-date="2019-11-27" data-time="15:11:40" data-timezone="UTC">03:11PM - 27 Nov 19 UTC</span>
      </div>

        <div class="date">
          closed <span class="discourse-local-date" data-format="ll" data-date="2019-12-12" data-time="23:28:06" data-timezone="UTC">11:28PM - 12 Dec 19 UTC</span>
        </div>

      <div class="user">
        <a href="https://github.com/lironmo" target="_blank" rel="noopener nofollow ugc">
          <img alt="lironmo" src="https://avatars.githubusercontent.com/u/37513762?v=4" class="onebox-avatar-inline" width="20" height="20">
          lironmo
        </a>
      </div>
    </div>

    <div class="labels">
        <span style="display:inline-block;margin-top:2px;background-color: #B8B8B8;padding: 2px;border-radius: 4px;color: #fff;margin-left: 3px;">
          oncall: jit
        </span>
        <span style="display:inline-block;margin-top:2px;background-color: #B8B8B8;padding: 2px;border-radius: 4px;color: #fff;margin-left: 3px;">
          module: onnx
        </span>
        <span style="display:inline-block;margin-top:2px;background-color: #B8B8B8;padding: 2px;border-radius: 4px;color: #fff;margin-left: 3px;">
          triaged
        </span>
    </div>
  </div>
</div>

  <div class="github-row">
    <p class="github-body-container">## 🐛 Bug


when convert scriptModule to onnx we crash and get the following e<span class="show-more-container"><a href="" rel="noopener" class="show-more">…</a></span><span class="excerpt hidden">xception:
```
Traceback (most recent call last):
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/utils.py", line 382, in _export
    fixed_batch_size=fixed_batch_size)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/utils.py", line 262, in _model_to_graph
    fixed_batch_size=fixed_batch_size)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/utils.py", line 132, in _optimize_graph
    graph = torch._C._jit_pass_onnx(graph, operator_export_type)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/__init__.py", line 174, in _run_symbolic_function
    return utils._run_symbolic_function(*args, **kwargs)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/utils.py", line 619, in _run_symbolic_function
    return op_fn(g, *inputs, **attrs)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/symbolic_helper.py", line 124, in wrapper
    return fn(g, *args)
  File "/home/liron/envs/detectron/lib/python3.6/site-packages/torch/onnx/symbolic_opset9.py", line 862, in batch_norm
    if len(input_sizes) == 2:
TypeError: object of type 'NoneType' has no len() (occurred when translating batch_norm)
```

## To Reproduce
load the attached torch script, and try to convert to onnx:

    def convert(self):
        loaded = torch.jit.load(self._torch_script_path)
        #loaded.load_state_dict(self._model_state)
        dummy_input = torch.randn(1, 3, 224, 224)
        target = loaded(dummy_input)
        torch.onnx.export(loaded, dummy_input, self._out_onnx_path, verbose=True,
        operator_export_type=torch.onnx.OperatorExportTypes.ONNX,
        example_outputs=target)
 

cc @suo @houseroad @spandantiwari @lara-hdr @BowenBao @neginraoof</span></p>
  </div>

  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>
<p>
However after reading these it’s not clear to me if <code>torch.jit.trace</code> → <code>ScriptModule</code> → <code>onnx</code> is supported.</p>
<p>— Edit 1 —</p>
<p>Based on these:</p><aside class="quote" data-post="17" data-topic="59754">
  <div class="title">
    <div class="quote-controls"></div>
    <img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/jerryzh168/48/15217_2.png" class="avatar">
    <a href="https://discuss.pytorch.org/t/onnx-export-failed-int8-model/59754/17">Onnx export failed int8 model</a> <a class="badge-category__wrapper " href="/c/quantization/17"><span data-category-id="17" data-drop-close="true" class="badge-category " title="This category is for questions, discussion and issues related to PyTorch’s quantization feature."><span class="badge-category__name">quantization</span></span></a>
  </div>
  <blockquote>
    what is the error message? i think linear is supported according to <a href="https://github.com/pytorch/pytorch/blob/master/torch/onnx/symbolic_caffe2.py" rel="noopener nofollow ugc">https://github.com/pytorch/pytorch/blob/master/torch/onnx/symbolic_caffe2.py</a>
  </blockquote>
</aside>
<p><a href="https://github.com/pytorch/pytorch/blob/master/test/onnx/test_pytorch_onnx_caffe2_quantized.py#L17-L35" class="onebox" target="_blank" rel="noopener nofollow ugc">https://github.com/pytorch/pytorch/blob/master/test/onnx/test_pytorch_onnx_caffe2_quantized.py#L17-L35</a></p>
<p>I tried this:</p>
<pre><code class="lang-auto">    # save the model as a PyTorch graph
    torch.save(mnistNet.state_dict(), 'MNIST.pt')

    model = mnistNet
    dummyInput = torch.randn(BATCH_SIZE, 1, IMAGE_WIDTH, IMAGE_HEIGHT).to('cpu')
    sample_inputs = dummyInput
    input_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

    torch.backends.quantized.engine = "qnnpack"

    sample_inputs = sample_inputs.numpy()
    pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs)
    model.qconfig = torch.quantization.get_default_qconfig('qnnpack')
    q_model = torch.quantization.prepare(model, inplace=False)
    q_model = torch.quantization.convert(q_model, inplace=False)

    traced_model = torch.jit.trace(q_model, pt_inputs)
    buf = io.BytesIO()
    torch.jit.save(traced_model, buf)
    buf.seek(0)
    q_model = torch.jit.load(buf)

    q_model.eval()
    output = q_model(*pt_inputs)

    f = io.BytesIO()
    torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, example_outputs=output,
                      operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK)
    f.seek(0)
</code></pre>
<p>Which produces:</p>
<pre><code class="lang-auto">$ python3 MNIST6.py 
beginning training . . .
epoch 1, epochLoss = 0.8135, epochAccuracy = 75.1496%
epoch 2, epochLoss = 0.2094, epochAccuracy = 93.6835%
epoch 3, epochLoss = 0.1284, epochAccuracy = 96.0439%
epoch 4, epochLoss = 0.0989, epochAccuracy = 97.0412%
epoch 5, epochLoss = 0.0763, epochAccuracy = 97.7560%
/usr/local/lib/python3.8/dist-packages/torch/quantization/observer.py:955: UserWarning: must run observer before calling calculate_qparams.                                    Returning default scale and zero point 
  warnings.warn(
Traceback (most recent call last):
  File "MNIST6.py", line 260, in &lt;module&gt;
    main()
  File "MNIST6.py", line 192, in main
    q_model = torch.quantization.convert(q_model, inplace=False)
  File "/usr/local/lib/python3.8/dist-packages/torch/quantization/quantize.py", line 471, in convert
    _convert(
  File "/usr/local/lib/python3.8/dist-packages/torch/quantization/quantize.py", line 509, in _convert
    reassign[name] = swap_module(mod, mapping, custom_module_class_mapping)
  File "/usr/local/lib/python3.8/dist-packages/torch/quantization/quantize.py", line 534, in swap_module
    new_mod = mapping[type(mod)].from_float(mod)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/quantized/modules/conv.py", line 418, in from_float
    return _ConvNd.from_float(cls, mod)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/quantized/modules/conv.py", line 220, in from_float
    return cls.get_qconv(mod, activation_post_process, weight_post_process)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/quantized/modules/conv.py", line 187, in get_qconv
    qweight = _quantize_weight(mod.weight.float(), weight_post_process)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/quantized/modules/utils.py", line 9, in _quantize_weight
    qweight = torch.quantize_per_tensor(
RuntimeError: quantize_tensor_per_tensor_affine expects a quantized and float tensors to be on the same device.
</code></pre>
<p>Any suggestions?  It’s not really clear to me if converting to onnx is supported after using <code>quantize_dynamic</code> or <code>optimize_for_mobile</code> first.  Is there a way around these errors?  Or is there an alternative way to make a PyTorch saved model much smaller?</p>

username: supriyar
cooked: <p>We currently do not fully support converting a pytorch quantized model to ONNX format. The current support is limited to running a subset of operators in Caffe2 via ONNX (converted from PT).</p>
<p>You could try checking out quantization in the ONNX framework, maybe they have ways to convert a quantized PT model to ONNX format.</p>