can't diarize my wav file following your tutorial
State: closed | Pull request: no

**Describe the bug**

Hey, I'm new to NeMo library, I want to do speaker diarization without ground truth (unsupervised), but I didn't manage to get the speaker diarization. I followed your speaker diarization tutorial here: 
https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/speaker_tasks/Speaker_Diarization_Inference.ipynb#scrollTo=0clD79kWkPlG
And tried the 'Clustering Diarizer: with System VAD' section to diarize my .wav file since I don't have ground truth labels, but I only get the data inference as embeddings in the output files.

**Steps/Code to reproduce bug**

Below is the code I used from your tutorial with my .wav path

import os
import wget
import matplotlib as plt
from nemo.collections.asr.parts.utils.speaker_utils import rttm_to_labels, labels_to_pyannote_object
from omegaconf import OmegaConf
import json
from nemo.collections.asr.models import ClusteringDiarizer
from nemo.collections.asr.parts.utils.vad_utils import plot

ROOT = os.getcwd()
data_dir = os.path.join(ROOT, 'data')
os.makedirs(data_dir, exist_ok=True)
#an4_audio = os.path.join(data_dir, 'an4_diarize_test.wav')
an4_audio = os.path.join(data_dir, 'output.wav') # placed my wav here

an4_rttm = os.path.join(data_dir, 'an4_diarize_test.rttm')
if not os.path.exists(an4_audio):
    an4_audio_url = "https://nemo-public.s3.us-east-2.amazonaws.com/an4_diarize_test.wav"
    an4_audio = wget.download(an4_audio_url, data_dir)
if not os.path.exists(an4_rttm):
    an4_rttm_url = "https://nemo-public.s3.us-east-2.amazonaws.com/an4_diarize_test.rttm"
    an4_rttm = wget.download(an4_rttm_url, data_dir)

labels = rttm_to_labels(an4_rttm)
reference = labels_to_pyannote_object(labels)
print(labels)
print(reference)

meta = {
    'audio_filepath': an4_audio,
    'offset': 0,
    'duration': None,
    'label': 'infer',
    'text': '-',
    'num_speakers': 2,
    'rttm_filepath': an4_rttm,
    'uem_filepath': None
}
with open('data/input_manifest.json', 'w') as fp:
    json.dump(meta, fp)
    fp.write('\n')

output_dir = os.path.join(ROOT, 'oracle_vad')
os.makedirs(output_dir, exist_ok=True)

MODEL_CONFIG = os.path.join(data_dir, 'diar_infer_telephonic.yaml')
if not os.path.exists(MODEL_CONFIG):
    config_url = "https://raw.githubusercontent.com/NVIDIA/NeMo/main/examples/speaker_tasks/diarization/conf/inference/diar_infer_telephonic.yaml"
    MODEL_CONFIG = wget.download(config_url, data_dir)

config = OmegaConf.load(MODEL_CONFIG)
print(OmegaConf.to_yaml(config))

config.diarizer.manifest_filepath = 'data/input_manifest.json'
config.diarizer.out_dir = output_dir  # Directory to store intermediate files and prediction outputs
pretrained_speaker_model = 'titanet_large'
config.diarizer.speaker_embeddings.model_path = pretrained_speaker_model
config.diarizer.speaker_embeddings.parameters.window_length_in_sec = [1.5, 1.25, 1.0, 0.75, 0.5]
config.diarizer.speaker_embeddings.parameters.shift_length_in_sec = [0.75, 0.625, 0.5, 0.375, 0.1]
config.diarizer.speaker_embeddings.parameters.multiscale_weights = [1, 1, 1, 1, 1]
config.diarizer.oracle_vad = True  # ----> ORACLE VAD
config.diarizer.clustering.parameters.oracle_num_speakers = False

oracle_vad_clusdiar_model = ClusteringDiarizer(cfg=config)

# And lets diarize
oracle_vad_clusdiar_model.diarize()

print("Clustering Diarizer Result (RTTM format)")
pred_labels_neural = rttm_to_labels(f'{output_dir}/pred_rttms/an4_diarize_test.rttm')
hypothesis_neural = labels_to_pyannote_object(pred_labels_neural)
print(hypothesis_neural)

print("Ground-truth Speaker Labels")
print(reference)

######################################################################
# no labels below:
print(OmegaConf.to_yaml(config))
pretrained_vad = 'vad_multilingual_marblenet'
pretrained_speaker_model = 'titanet_large'

config.num_workers = 1  # Workaround for multiprocessing hanging with ipython issue

output_dir = os.path.join(ROOT, 'outputs')
config.diarizer.manifest_filepath = 'data/input_manifest.json'
config.diarizer.out_dir = output_dir  # Directory to store intermediate files and prediction outputs

config.diarizer.speaker_embeddings.model_path = pretrained_speaker_model
config.diarizer.oracle_vad = False  # compute VAD provided with model_path to vad config
config.diarizer.clustering.parameters.oracle_num_speakers = False

# Here, we use our in-house pretrained NeMo VAD model
config.diarizer.vad.model_path = pretrained_vad
config.diarizer.vad.parameters.onset = 0.8
config.diarizer.vad.parameters.offset = 0.6
config.diarizer.vad.parameters.pad_offset = -0.05

sd_model = ClusteringDiarizer(cfg=config)

sd_model.diarize()

# VAD predicted time stamps
# you can also use single threshold(=onset=offset) for binarization and plot here


if config.diarizer.vad.parameters.smoothing:
    vad_output_filepath = f'{output_dir}/vad_outputs/overlap_smoothing_output_median_{config.diarizer.vad.parameters.overlap}/an4_diarize_test.{config.diarizer.vad.parameters.smoothing}'
else:
    vad_output_filepath = f'{output_dir}/vad_outputs/an4_diarize_test.frame'


**Expected behavior**

A clear and concise description of what you expected to happen.

**Environment overview (please complete the following information)**

 - Environment location: [Bare-metal, Docker, Cloud(specify cloud provider - AWS, Azure, GCP, Collab)]
 - Method of NeMo install: [pip install or from source]. Please specify exact commands you used to install.
 - If method of install is [Docker], provide `docker pull` & `docker run` commands used

**Environment details**

- Using ubuntu 20.04.6
- PyTorch 2.0.1
- Python 3.8.10

What am I missing? Any help would be appreciated!