Unable to load MarbleNet for VAD
State: closed | Pull request: no
Labels: name: bug

name: stale

I am trying to follow the tutorial available for VAD using the below snippet. However, I am not available to load MarbleNet model.

```
import nemo
import nemo.collections.asr as nemo_asr
vad_model = nemo_asr.models.EncDecClassificationModel.from_pretrained('vad_marblenet')
```

Following is the traceback:

```
    Error(s):	Error in call to target 'nemo.collections.asr.modules.audio_preprocessing.SpectrogramAugmentation':
    AttributeError('/usr/local/cuda/nvvm/lib64/libnvvm.so.3.1.0: undefined symbol: nvvmLazyAddModuleToProgram')
    Traceback (most recent call last):
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 92, in _call_target
        return _target_(*args, **kwargs)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/modules/audio_preprocessing.py", line 512, in __init__
        if use_numba_spec_augment and numba_utils.numba_cuda_is_supported(__NUMBA_MINIMUM_VERSION__):
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/utils/numba_utils.py", line 129, in numba_cuda_is_supported
        cuda_available = cuda.is_available()
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/device_init.py", line 60, in is_available
        return driver_is_available and nvvm.is_available()
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/cudadrv/nvvm.py", line 56, in is_available
        NVVM()
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/cudadrv/nvvm.py", line 135, in __new__
        func = getattr(inst.driver, name)
      File "/opt/conda/envs/VAD_test/lib/python3.8/ctypes/__init__.py", line 386, in __getattr__
        func = self.__getitem__(name)
      File "/opt/conda/envs/VAD_test/lib/python3.8/ctypes/__init__.py", line 391, in __getitem__
        func = self._FuncPtr((name_or_ordinal, self))
    AttributeError: /usr/local/cuda/nvvm/lib64/libnvvm.so.3.1.0: undefined symbol: nvvmLazyAddModuleToProgram
```
    
```
    The above exception was the direct cause of the following exception:
    
    Traceback (most recent call last):
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py", line 485, in from_config_dict
        instance = imported_cls(cfg=config, trainer=trainer)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/models/classification_models.py", line 443, in __init__
        super().__init__(cfg=cfg, trainer=trainer)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/models/classification_models.py", line 63, in __init__
        self.spec_augmentation = ASRModel.from_config_dict(self._cfg.spec_augment)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py", line 468, in from_config_dict
        instance = hydra.utils.instantiate(config=config)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 222, in instantiate
        return instantiate_node(
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 339, in instantiate_node
        return _call_target(_target_, partial, args, kwargs, full_key)
      File "/opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py", line 97, in _call_target
        raise InstantiationException(msg) from e
    hydra.errors.InstantiationException: Error in call to target 'nemo.collections.asr.modules.audio_preprocessing.SpectrogramAugmentation':
    AttributeError('/usr/local/cuda/nvvm/lib64/libnvvm.so.3.1.0: undefined symbol: nvvmLazyAddModuleToProgram')
    
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py:92, in _call_target(_target_, _partial_, args, kwargs, full_key)
     91 try:
---> 92     return _target_(*args, **kwargs)
     93 except Exception as e:

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/modules/audio_preprocessing.py:512, in SpectrogramAugmentation.__init__(self, freq_masks, time_masks, freq_width, time_width, rect_masks, rect_time, rect_freq, rng, mask_value, use_numba_spec_augment)
    511 # Check if numba is supported, and use a Numba kernel if it is
--> 512 if use_numba_spec_augment and numba_utils.numba_cuda_is_supported(__NUMBA_MINIMUM_VERSION__):
    513     logging.info('Numba CUDA SpecAugment kernel is being used')

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/utils/numba_utils.py:129, in numba_cuda_is_supported(min_version)
    128 try:
--> 129     cuda_available = cuda.is_available()
    130     if cuda_available:

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/device_init.py:60, in is_available()
     58     pass
---> 60 return driver_is_available and nvvm.is_available()

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/cudadrv/nvvm.py:56, in is_available()
     55 try:
---> 56     NVVM()
     57 except NvvmSupportError:

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/cudadrv/nvvm.py:143, in NVVM.__init__(self)
    142 def __init__(self):
--> 143     ir_versions = self.get_ir_version()
    144     self._majorIR = ir_versions[0]

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/numba/cuda/cudadrv/nvvm.py:168, in NVVM.get_ir_version(self)
    167 minorDbg = c_int()
--> 168 err = self.nvvmIRVersion(byref(majorIR), byref(minorIR),
    169                          byref(majorDbg), byref(minorDbg))
    170 self.check_error(err, 'Failed to get IR version.')

AttributeError: 'NVVM' object has no attribute 'nvvmIRVersion'

The above exception was the direct cause of the following exception:

InstantiationException                    Traceback (most recent call last)
Cell In[5], line 1
----> 1 vad_model = nemo_asr.models.EncDecClassificationModel.from_pretrained('vad_marblenet')

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py:852, in Model.from_pretrained(cls, model_name, refresh_cache, override_config_path, map_location, strict, return_config, trainer, save_restore_connector)
    846 else:
    847     # NGC source
    848     class_, nemo_model_file_in_cache = cls._get_ngc_pretrained_model_info(
    849         model_name=model_name, refresh_cache=refresh_cache
    850     )
--> 852 instance = class_.restore_from(
    853     restore_path=nemo_model_file_in_cache,
    854     override_config_path=override_config_path,
    855     map_location=map_location,
    856     strict=strict,
    857     return_config=return_config,
    858     trainer=trainer,
    859     save_restore_connector=save_restore_connector,
    860 )
    861 return instance

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/modelPT.py:330, in ModelPT.restore_from(cls, restore_path, override_config_path, map_location, strict, return_config, save_restore_connector, trainer)
    327 app_state.model_restore_path = restore_path
    329 cls.update_save_restore_connector(save_restore_connector)
--> 330 instance = cls._save_restore_connector.restore_from(
    331     cls, restore_path, override_config_path, map_location, strict, return_config, trainer
    332 )
    333 if isinstance(instance, ModelPT):
    334     instance._save_restore_connector = save_restore_connector

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/connectors/save_restore_connector.py:235, in SaveRestoreConnector.restore_from(self, calling_cls, restore_path, override_config_path, map_location, strict, return_config, trainer)
    210 """
    211 Restores model instance (weights and configuration) into .nemo file
    212 
   (...)
    231     An instance of type cls or its underlying config (if return_config is set).
    232 """
    233 # Get path where the command is executed - the artifacts will be "retrieved" there
    234 # (original .nemo behavior)
--> 235 loaded_params = self.load_config_and_state_dict(
    236     calling_cls, restore_path, override_config_path, map_location, strict, return_config, trainer,
    237 )
    238 if not isinstance(loaded_params, tuple):
    239     return loaded_params

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/connectors/save_restore_connector.py:158, in SaveRestoreConnector.load_config_and_state_dict(self, calling_cls, restore_path, override_config_path, map_location, strict, return_config, trainer)
    156 # get the class
    157 calling_cls._set_model_restore_state(is_being_restored=True, folder=tmpdir)
--> 158 instance = calling_cls.from_config_dict(config=conf, trainer=trainer)
    159 instance = instance.to(map_location)
    160 # add load_state_dict override

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py:507, in Serialization.from_config_dict(cls, config, trainer)
    505             if prev_error:
    506                 logging.error(prev_error)
--> 507             raise e
    509 if not hasattr(instance, '_cfg'):
    510     instance._cfg = config

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py:499, in Serialization.from_config_dict(cls, config, trainer)
    497 accepts_trainer = Serialization._inspect_signature_for_trainer(cls)
    498 if accepts_trainer:
--> 499     instance = cls(cfg=config, trainer=trainer)
    500 else:
    501     instance = cls(cfg=config)

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/models/classification_models.py:443, in EncDecClassificationModel.__init__(self, cfg, trainer)
    440 if cfg.get("is_regression_task", False):
    441     raise ValueError(f"EndDecClassificationModel requires the flag is_regression_task to be set as false")
--> 443 super().__init__(cfg=cfg, trainer=trainer)

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/collections/asr/models/classification_models.py:63, in _EncDecBaseModel.__init__(self, cfg, trainer)
     60 super().__init__(cfg=cfg, trainer=trainer)
     62 if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None:
---> 63     self.spec_augmentation = ASRModel.from_config_dict(self._cfg.spec_augment)
     64 else:
     65     self.spec_augmentation = None

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/nemo/core/classes/common.py:468, in Serialization.from_config_dict(cls, config, trainer)
    465 # Hydra 1.x API
    466 elif '_target_' in config and _HAS_HYDRA:
    467     # regular hydra-based instantiation
--> 468     instance = hydra.utils.instantiate(config=config)
    469 else:
    470     instance = None

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py:222, in instantiate(config, *args, **kwargs)
    219     _convert_ = config.pop(_Keys.CONVERT, ConvertMode.NONE)
    220     _partial_ = config.pop(_Keys.PARTIAL, False)
--> 222     return instantiate_node(
    223         config, *args, recursive=_recursive_, convert=_convert_, partial=_partial_
    224     )
    225 elif OmegaConf.is_list(config):
    226     # Finalize config (convert targets to strings, merge with kwargs)
    227     config_copy = copy.deepcopy(config)

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py:339, in instantiate_node(node, convert, recursive, partial, *args)
    334                 value = instantiate_node(
    335                     value, convert=convert, recursive=recursive
    336                 )
    337             kwargs[key] = _convert_node(value, convert)
--> 339     return _call_target(_target_, partial, args, kwargs, full_key)
    340 else:
    341     # If ALL or PARTIAL non structured, instantiate in dict and resolve interpolations eagerly.
    342     if convert == ConvertMode.ALL or (
    343         convert == ConvertMode.PARTIAL
    344         and node._metadata.object_type in (None, dict)
    345     ):

File /opt/conda/envs/VAD_test/lib/python3.8/site-packages/hydra/_internal/instantiate/_instantiate2.py:97, in _call_target(_target_, _partial_, args, kwargs, full_key)
     95 if full_key:
     96     msg += f"\nfull_key: {full_key}"
---> 97 raise InstantiationException(msg) from e

InstantiationException: Error in call to target 'nemo.collections.asr.modules.audio_preprocessing.SpectrogramAugmentation':
AttributeError("'NVVM' object has no attribute 'nvvmIRVersion'")


```
**Environment details**
python 3.8.13
pytorch-lightning        1.8.6
torch                    1.13.1
torchmetrics             0.11.1
nemo-toolkit             1.15.0