Exception running on cpu
AutoAWQ installed, cudatoolkit installed.
as per instrucions.
Traceback (most recent call last):
File "/usr/local/apps/mistral-awq/main.py", line 1, in
from awq import AutoAWQForCausalLM
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/init.py", line 2, in
from awq.models.auto import AutoAWQForCausalLM
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/models/init.py", line 1, in
from .mpt import MptAWQForCausalLM
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/models/mpt.py", line 1, in
from .base import BaseAWQForCausalLM
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/models/base.py", line 11, in
from awq.quantize.quantizer import AwqQuantizer
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/quantize/quantizer.py", line 11, in
from awq.modules.linear import WQLinear_GEMM, WQLinear_GEMV
File "/root/miniconda3/envs/mistral/lib/python3.11/site-packages/awq/modules/linear.py", line 4, in
import awq_inference_engine # with CUDA kernels
^^^^^^^^^^^^^^^^^^^^^^^^^^^
ImportError: libcudart.so.11.0: cannot open shared object file: No such file or directory