montreal_forced_aligner/utils.py
def thirdparty_binary(binary_name: str) -> str:
    """
    Generate full path to a given binary name

    Notes
    -----
    With the move to conda, this function is deprecated as conda will manage the path much better

    Parameters
    ----------
    binary_name: str
        Executable to run

    Returns
    -------
    str
        Full path to the executable
    """
    bin_path = shutil.which(binary_name)
    if bin_path is None:
        if binary_name in ["fstcompile", "fstarcsort", "fstconvert"] and sys.platform != "win32":
            raise ThirdpartyError(binary_name, open_fst=True)
        else:
            raise ThirdpartyError(binary_name)
    return bin_path
montreal_forced_aligner/corpus/features.py
class ComputeVadFunction(KaldiFunction):
    """
    Multiprocessing function to compute voice activity detection

    See Also
    --------
    :meth:`.AcousticCorpusMixin.compute_vad`
        Main function that calls this function in parallel
    :meth:`.AcousticCorpusMixin.compute_vad_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`compute-vad`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.corpus.features.VadArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*processed (?P<done>\d+) utterances.*(?P<no_feats>\d+) had.*(?P<unvoiced>\d+) were.*"
    )

    def __init__(self, args: VadArguments):
        self.log_path = args.log_path
        self.feats_scp_path = args.feats_scp_path
        self.vad_scp_path = args.vad_scp_path
        self.vad_options = args.vad_options

    def run(self):
        """Run the function"""
        with open(self.log_path, "w") as log_file:
            feats_scp_path = self.feats_scp_path
            vad_scp_path = self.vad_scp_path
            vad_proc = subprocess.Popen(
                [
                    thirdparty_binary("compute-vad"),
                    f"--vad-energy-mean-scale={self.vad_options['energy_mean_scale']}",
                    f"--vad-energy-threshold={self.vad_options['energy_threshold']}",
                    f"scp:{feats_scp_path}",
                    f"ark,t:{vad_scp_path}",
                ],
                stderr=subprocess.PIPE,
                encoding="utf8",
                env=os.environ,
            )
            for line in vad_proc.stderr:
                log_file.write(line)
                m = self.progress_pattern.match(line.strip())
                if m:
                    yield int(m.group("done")), int(m.group("no_feats")), int(m.group("unvoiced"))
montreal_forced_aligner/textgrid.py
def parse_aligned_textgrid(
    path: str, root_speaker: typing.Optional[str] = None
) -> Dict[str, List[CtmInterval]]:
    tg = tgio.openTextgrid(path, includeEmptyIntervals=False, reportingMode="silence")
    data = {}
    num_tiers = len(tg.tierNameList)
    if num_tiers == 0:
        raise TextGridParseError(path, "Number of tiers parsed was zero")
    phone_tier_pattern = re.compile(r"(.*) ?- ?phones")
    for tier_name in tg.tierNameList:
        ti = tg.tierDict[tier_name]
        if not isinstance(ti, tgio.IntervalTier):
            continue
        if "phones" not in tier_name:
            continue
        m = phone_tier_pattern.match(tier_name)
        if m:
            speaker_name = m.groups()[0]
        elif root_speaker:
            speaker_name = root_speaker
        else:
            speaker_name = ""
        if speaker_name not in data:
            data[speaker_name] = []
        for begin, end, text in ti.entryList:
            text = text.lower().strip()
            if not text:
                continue
            begin, end = round(begin, 4), round(end, 4)
            if end - begin < 0.01:
                continue
            interval = CtmInterval(begin, end, text, speaker_name)
            data[speaker_name].append(interval)
    return data
montreal_forced_aligner/helper.py
def load_scp(path: str, data_type: Optional[Type] = str) -> CorpusMappingType:
    """
    Load a Kaldi script file (.scp)

    Scp files in Kaldi can either be one-to-one or one-to-many, with the first element separated by
    whitespace as the key and the remaining whitespace-delimited elements the values.

    Returns a dictionary of key to value for
    one-to-one mapping case and a dictionary of key to list of values for one-to-many case.

    See Also
    --------
    :kaldi_docs:`io#io_sec_scp_details`
        For more information on the SCP format

    Parameters
    ----------
    path : str
        Path to Kaldi script file
    data_type : type
        Type to coerce the data to

    Returns
    -------
    CorpusMappingType
        Dictionary where the keys are the first column and the values are all
        other columns in the scp file

    """
    scp = {}
    with open(path, "r", encoding="utf8") as f:
        for line in f:
            line = line.strip()
            if line == "":
                continue
            line_list = line.split()
            key = load_scp_safe(line_list.pop(0))
            if len(line_list) == 1:
                value = data_type(line_list[0])
                if isinstance(value, str):
                    value = load_scp_safe(value)
            else:
                value = [data_type(x) for x in line_list if x not in ["[", "]"]]
            scp[key] = value
    return scp
montreal_forced_aligner/corpus/helper.py
def find_exts(files: typing.List[str]) -> FileExtensions:
    """
    Find and group sound file extensions and transcription file extensions

    Parameters
    ----------
    files: List
        List of filename

    Returns
    -------
    :class:`~montreal_forced_aligner.data.FileExtensions`
        Data class for files found
    """
    exts = FileExtensions([], {}, {}, {}, {})
    for full_filename in files:
        filename, fext = os.path.splitext(full_filename)
        fext = fext.lower()
        if fext == ".wav":
            exts.wav_files[filename] = full_filename
        elif fext == ".lab":
            exts.lab_files[filename] = full_filename
        elif (
            fext == ".txt" and filename not in exts.lab_files
        ):  # .lab files have higher priority than .txt files
            exts.lab_files[filename] = full_filename
        elif fext == ".textgrid":
            exts.textgrid_files[filename] = full_filename
        elif fext in supported_audio_extensions and shutil.which("sox") is not None:
            exts.other_audio_files[filename] = full_filename
        if filename not in exts.identifiers:
            exts.identifiers.append(filename)
    return exts
montreal_forced_aligner/abc.py
class TemporaryDirectoryMixin(metaclass=abc.ABCMeta):
    """
    Abstract mixin class for MFA temporary directories

    Parameters
    ----------
    temporary_directory: str, optional
        Path to store temporary files
    """

    def __init__(
        self,
        temporary_directory: str = None,
        **kwargs,
    ):
        super().__init__(**kwargs)
        if not temporary_directory:
            from .config import get_temporary_directory

            temporary_directory = get_temporary_directory()
        self.temporary_directory = temporary_directory
        self._corpus_output_directory = None
        self._dictionary_output_directory = None
        self._language_model_output_directory = None

    @property
    @abc.abstractmethod
    def identifier(self) -> str:
        """Identifier to use in creating the temporary directory"""
        ...

    @property
    @abc.abstractmethod
    def data_source_identifier(self) -> str:
        """Identifier for the data source (generally the corpus being used)"""
        ...

    @property
    @abc.abstractmethod
    def output_directory(self) -> str:
        """Root temporary directory"""
        ...

    @property
    def corpus_output_directory(self) -> str:
        """Temporary directory containing all corpus information"""
        if self._corpus_output_directory:
            return self._corpus_output_directory
        return os.path.join(self.output_directory, f"{self.data_source_identifier}")

    @corpus_output_directory.setter
    def corpus_output_directory(self, directory: str) -> None:
        self._corpus_output_director = directory

    @property
    def dictionary_output_directory(self) -> str:
        """Temporary directory containing all dictionary information"""
        if self._dictionary_output_directory:
            return self._dictionary_output_directory
        return os.path.join(self.output_directory, "dictionary")

    @dictionary_output_directory.setter
    def dictionary_output_directory(self, directory: str) -> None:
        self._dictionary_output_directory = directory

    @property
    def language_model_output_directory(self) -> str:
        """Temporary directory containing all dictionary information"""
        if self._language_model_output_directory:
            return self._language_model_output_directory
        return os.path.join(self.output_directory, "language_model")

    @language_model_output_directory.setter
    def language_model_output_directory(self, directory: str) -> None:
        self._language_model_output_directory = directory
montreal_forced_aligner/abc.py
class MfaWorker(metaclass=abc.ABCMeta):
    """
    Abstract class for MFA workers

    Parameters
    ----------
    use_mp: bool
        Flag to run in multiprocessing mode, defaults to True
    debug: bool
        Flag to run in debug mode, defaults to False
    verbose: bool
        Flag to run in verbose mode, defaults to False

    Attributes
    ----------
    dirty: bool
        Flag for whether an error was encountered in processing
    """

    def __init__(
        self,
        use_mp: bool = True,
        debug: bool = False,
        verbose: bool = False,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.debug = debug
        self.verbose = verbose
        self.use_mp = use_mp
        self.dirty = False

    def log_debug(self, message: str) -> None:
        """
        Print a debug message

        Parameters
        ----------
        message: str
            Debug message to log
        """
        print(message)

    def log_error(self, message: str) -> None:
        """
        Print an error message

        Parameters
        ----------
        message: str
            Error message to log
        """
        print(message)

    def log_info(self, message: str) -> None:
        """
        Print an info message

        Parameters
        ----------
        message: str
            Info message to log
        """
        print(message)

    def log_warning(self, message: str) -> None:
        """
        Print a warning message

        Parameters
        ----------
        message: str
            Warning message to log
        """
        print(message)

    @classmethod
    def extract_relevant_parameters(cls, config: MetaDict) -> Tuple[MetaDict, List[str]]:
        """
        Filter a configuration dictionary to just the relevant parameters for the current worker

        Parameters
        ----------
        config: dict[str, Any]
            Configuration dictionary

        Returns
        -------
        dict[str, Any]
            Filtered configuration dictionary
        list[str]
            Skipped keys
        """
        skipped = []
        new_config = {}
        for k, v in config.items():
            if k in cls.get_configuration_parameters():
                new_config[k] = v
            else:
                skipped.append(k)
        return new_config, skipped

    @classmethod
    def get_configuration_parameters(cls) -> Dict[str, Type]:
        """
        Get the types of parameters available to be configured

        Returns
        -------
        dict[str, Type]
            Dictionary of parameter names and their types
        """
        mapping = {Dict: dict, Tuple: tuple, List: list, Set: set}
        configuration_params = {}
        for t, ty in get_type_hints(cls.__init__).items():
            configuration_params[t] = ty
            try:
                if ty.__origin__ == Union:
                    configuration_params[t] = ty.__args__[0]
            except AttributeError:
                pass

        for c in cls.mro():
            try:
                for t, ty in get_type_hints(c.__init__).items():
                    configuration_params[t] = ty
                    try:
                        if ty.__origin__ == Union:
                            configuration_params[t] = ty.__args__[0]
                    except AttributeError:
                        pass
            except AttributeError:
                pass
        for t, ty in configuration_params.items():
            for v in mapping.values():
                try:
                    if ty.__origin__ == v:
                        configuration_params[t] = v
                        break
                except AttributeError:
                    break
        return configuration_params

    @property
    def configuration(self) -> MetaDict:
        """Configuration parameters"""
        return {
            "debug": self.debug,
            "verbose": self.verbose,
            "use_mp": self.use_mp,
            "dirty": self.dirty,
        }

    @property
    @abc.abstractmethod
    def working_directory(self) -> str:
        """Current working directory"""
        ...

    @property
    def working_log_directory(self) -> str:
        """Current working log directory"""
        return os.path.join(self.working_directory, "log")

    @property
    @abc.abstractmethod
    def data_directory(self) -> str:
        """Data directory"""
        ...
montreal_forced_aligner/utils.py
class Stopped(object):
    """
    Multiprocessing class for detecting whether processes should stop processing and exit ASAP

    Attributes
    ----------
    val: :func:`~multiprocessing.Value`
        0 if not stopped, 1 if stopped
    lock: :class:`~multiprocessing.Lock`
        Lock for process safety
    _source: multiprocessing.Value
        1 if it was a Ctrl+C event that stopped it, 0 otherwise
    """

    def __init__(self, initval: Union[bool, int] = False):
        self.val = mp.Value("i", initval)
        self.lock = mp.Lock()
        self._source = mp.Value("i", 0)

    def stop(self) -> None:
        """Signal that work should stop asap"""
        with self.lock:
            self.val.value = True

    def stop_check(self) -> int:
        """Check whether a process should stop"""
        with self.lock:
            return self.val.value

    def set_sigint_source(self) -> None:
        """Set the source as a ctrl+c"""
        with self.lock:
            self._source.value = True

    def source(self) -> int:
        """Get the source value"""
        with self.lock:
            return self._source.value
montreal_forced_aligner/data.py
class TextFileType(enum.Enum):
    """Enum for types of text files"""

    NONE = 0
    TEXTGRID = 1
    LAB = 2
montreal_forced_aligner/corpus/classes.py
class File(MfaCorpusClass):
    """
    File class for representing metadata and associations of Files

    Parameters
    ----------
    wav_path: str, optional
        Sound file path
    text_path: str, optional
        Transcription file path
    relative_path: str, optional
        Relative path to the corpus root

    Attributes
    ----------
    utterances: :class:`~montreal_forced_aligner.corpus.classes.UtteranceCollection`
        Utterances in the file
    speaker_ordering: list[Speaker]
        Ordering of speakers in the transcription file
    wav_info: :class:`~montreal_forced_aligner.data.SoundFileInformation`
        Information about sound file
    waveform: numpy.array
        Audio samples
    aligned: bool
        Flag for whether a file has alignments

    Raises
    ------
    :class:`~montreal_forced_aligner.exceptions.CorpusError`
        If both wav_path and text_path are None
    """

    textgrid_regex = re.compile(r"\.textgrid$", flags=re.IGNORECASE)
    wav_regex = re.compile(r"\.wav$", flags=re.IGNORECASE)

    def __init__(
        self,
        wav_path: Optional[str] = None,
        text_path: Optional[str] = None,
        relative_path: Optional[str] = None,
        name: Optional[str] = None,
    ):
        self.wav_path = wav_path
        self.text_path = text_path
        wav_check = self.wav_path is not None
        text_check = self.text_path is not None
        self._name = name
        if not self._name:
            if wav_check:
                self._name = os.path.splitext(os.path.basename(self.wav_path))[0]
            elif text_check:
                self._name = os.path.splitext(os.path.basename(self.text_path))[0]
            else:
                raise CorpusError("File objects must have either a wav_path or text_path")
        self.relative_path = relative_path
        self.wav_info: Optional[SoundFileInformation] = None
        self.waveform = None
        self.speaker_ordering: List[Speaker] = []
        self.utterances = UtteranceCollection()
        self.aligned = False
        if text_check:
            if self.text_path.lower().endswith(".textgrid"):
                self.text_type = TextFileType.TEXTGRID
            else:
                self.text_type = TextFileType.LAB
        else:
            self.text_type = TextFileType.NONE
        if wav_check:

            if self.wav_path.lower().endswith(".wav"):
                self.sound_type = SoundFileType.WAV
            else:
                self.sound_type = SoundFileType.SOX
        else:
            self.sound_type = SoundFileType.NONE

    @property
    def multiprocessing_data(self) -> FileData:
        """
        Data object for the file
        """
        return FileData(
            self._name,
            self.wav_path,
            self.text_path,
            self.relative_path,
            self.wav_info,
            [s.name for s in self.speaker_ordering],
            [u.multiprocessing_data for u in self.utterances],
        )

    @classmethod
    def load_from_mp_data(cls, file_data: FileData) -> File:
        """
        Construct a File from a multiprocessing file data class

        Parameters
        ----------
        file_data: :class:`~montreal_forced_aligner.data.FileData`
            Data for the loaded file

        Returns
        -------
        :class:`~montreal_forced_aligner.corpus.classes.File`
            Loaded file
        """
        file = File(
            file_data.wav_path,
            file_data.text_path,
            relative_path=file_data.relative_path,
            name=file_data.name,
        )
        file.wav_info = file_data.wav_info
        for s in file_data.speaker_ordering:
            file.add_speaker(Speaker(s))
        for u in file_data.utterances:
            u = Utterance.load_from_mp_data(u, file)
            file.utterances.add_utterance(u)
        return file

    @classmethod
    def parse_file(
        cls,
        file_name: str,
        wav_path: Optional[str],
        text_path: Optional[str],
        relative_path: str,
        speaker_characters: Union[int, str],
        sanitize_function: Optional[MultispeakerSanitizationFunction] = None,
    ):
        """
        Parse a collection of sound file and transcription file into a File

        Parameters
        ----------
        file_name: str
            File identifier
        wav_path: str
            Full sound file path
        text_path: str
            Full transcription path
        relative_path: str
            Relative path from the corpus directory root
        speaker_characters: int, optional
            Number of characters in the file name to specify the speaker
        sanitize_function: Callable, optional
            Function to sanitize words and strip punctuation

        Returns
        -------
        :class:`~montreal_forced_aligner.corpus.classes.File`
            Parsed file
        """
        file = File(wav_path, text_path, relative_path=relative_path, name=file_name)
        if file.has_sound_file:
            root = os.path.dirname(wav_path)
            file.wav_info = get_wav_info(wav_path)
        else:
            root = os.path.dirname(text_path)
        if not speaker_characters:
            speaker_name = os.path.basename(root)
        elif isinstance(speaker_characters, int):
            speaker_name = file_name[:speaker_characters]
        elif speaker_characters == "prosodylab":
            speaker_name = file_name.split("_")[1]
        else:
            speaker_name = file_name
        root_speaker = None
        if speaker_characters or file.text_type != TextFileType.TEXTGRID:
            root_speaker = Speaker(speaker_name)
        file.load_text(
            root_speaker=root_speaker,
            sanitize_function=sanitize_function,
        )
        return file

    def __eq__(self, other: Union[File, str]) -> bool:
        """Check if a File is equal to another File"""
        if isinstance(other, File):
            return other.name == self.name
        if isinstance(other, str):
            return self.name == other
        raise TypeError("Files can only be compared to other files and strings.")

    def __lt__(self, other: Union[File, str]) -> bool:
        """Check if a File is less than another File"""
        if isinstance(other, File):
            return other.name < self.name
        if isinstance(other, str):
            return self.name < other
        raise TypeError("Files can only be compared to other files and strings.")

    def __gt__(self, other: Union[File, str]) -> bool:
        """Check if a File is greater than another File"""
        if isinstance(other, File):
            return other.name > self.name
        if isinstance(other, str):
            return self.name > other
        raise TypeError("Files can only be compared to other files and strings.")

    def __hash__(self) -> hash:
        """Get the hash of the file"""
        return hash(self.name)

    @property
    def name(self) -> str:
        """Name of the file"""
        name = self._name
        if self.relative_path:
            prefix = self.relative_path.replace("/", "").replace("\\", "")
            if not name.startswith(prefix):
                name = f"{prefix}_{name}"
        return name

    @property
    def is_fully_aligned(self) -> bool:
        """
        Check if all utterances have been aligned
        """
        for u in self.utterances:
            if u.ignored:
                continue
            if u.word_labels is None:
                return False
            if u.phone_labels is None:
                return False
        return True

    def __repr__(self) -> str:
        """Representation of File objects"""
        return f'<File {self.name} Sound path="{self.wav_path}" Text path="{self.text_path}">'

    def save(
        self,
        output_directory: Optional[str] = None,
        text_type: Optional[TextFileType] = None,
        save_transcription: bool = False,
    ) -> None:
        """
        Output File to TextGrid or lab.  If ``text_type`` is not specified, the original file type will be used,
        but if there was no text file for the file, it will guess lab format if there is only one utterance, otherwise
        it will output a TextGrid file.

        Parameters
        ----------
        output_directory: str, optional
            Directory to output file, if None, then it will overwrite the original file
        backup_output_directory: str, optional
            If specified, then it will check whether it would overwrite an existing file and
            instead use this directory
        text_type: TextFileType, optional
            Text type to save as, if not provided, it will use either the original file type or guess the file type
        save_transcription: bool
            Flag for whether the hypothesized transcription text should be saved instead of the default text
        """
        utterance_count = len(self.utterances)
        if text_type is None:
            text_type = self.text_type
            if text_type == TextFileType.NONE:
                if utterance_count == 1:
                    text_type = TextFileType.LAB
                else:
                    text_type = TextFileType.TEXTGRID
        if text_type == TextFileType.LAB:
            if utterance_count == 0 and os.path.exists(self.text_path) and not save_transcription:
                os.remove(self.text_path)
                return
            elif utterance_count == 0:
                return
            output_path = self.construct_output_path(output_directory, enforce_lab=True)
            with open(output_path, "w", encoding="utf8") as f:
                for u in self.utterances:
                    if save_transcription:
                        f.write(u.transcription_text if u.transcription_text else "")
                    else:
                        f.write(u.text)
            return
        elif text_type == TextFileType.TEXTGRID:
            output_path = self.construct_output_path(output_directory)
            max_time = self.duration
            tiers = {}
            for speaker in self.speaker_ordering:
                if speaker is None:
                    tiers["speech"] = textgrid.IntervalTier("speech", [], minT=0, maxT=max_time)
                else:
                    tiers[speaker] = textgrid.IntervalTier(speaker.name, [], minT=0, maxT=max_time)

            tg = textgrid.Textgrid()
            tg.maxTimestamp = max_time
            for utterance in self.utterances:

                if utterance.speaker is None:
                    speaker = "speech"
                else:
                    speaker = utterance.speaker
                if not self.aligned:

                    if save_transcription:
                        tiers[speaker].entryList.append(
                            Interval(
                                start=utterance.begin,
                                end=utterance.end,
                                label=utterance.transcription_text
                                if utterance.transcription_text
                                else "",
                            )
                        )
                    else:
                        tiers[speaker].entryList.append(
                            Interval(
                                start=utterance.begin, end=utterance.end, label=utterance.text
                            )
                        )
            for t in tiers.values():
                tg.addTier(t)
            tg.save(output_path, includeBlankSpaces=True, format="long_textgrid")

    @property
    def meta(self) -> Dict[str, Any]:
        """Metadata for the File"""
        return {
            "wav_path": self.wav_path,
            "text_path": self.text_path,
            "name": self._name,
            "relative_path": self.relative_path,
            "wav_info": self.wav_info.meta,
            "speaker_ordering": [x.name for x in self.speaker_ordering],
        }

    @property
    def has_sound_file(self) -> bool:
        """Flag for whether the File has a sound file"""
        return self.sound_type != SoundFileType.NONE

    @property
    def has_text_file(self) -> bool:
        """Flag for whether the File has a text file"""
        return self.text_type != TextFileType.NONE

    @property
    def aligned_data(self) -> Dict[str, Dict[str, List[CtmInterval]]]:
        """
        Word and phone alignments for the file

        Returns
        -------
        dict[str, dict[str, list[CtmInterval]]]
            Dictionary of word and phone intervals for each speaker in the file
        """
        data = {}
        for s in self.speaker_ordering:
            if s.name not in data:
                data[s.name] = {"words": [], "phones": []}
        for u in self.utterances:
            if u.word_labels is None:
                continue
            data[u.speaker_name]["words"].extend(u.word_labels)
            data[u.speaker_name]["phones"].extend(u.phone_labels)
        return data

    def clean_up(self) -> None:
        """
        Recombine words that were split up as part of initial text processing
        """
        for u in self.utterances:
            if not u.word_labels:
                continue
            cur_ind = 0
            actual_labels = []
            dictionary = u.speaker.dictionary
            for word in u.text.split():
                splits = dictionary.lookup(word)
                b = 1000000
                e = -1
                for w in splits:
                    cur = u.word_labels[cur_ind]
                    if w == cur.label or cur.label == dictionary.oov_word:
                        if cur.begin < b:
                            b = cur.begin
                        if cur.end > e:
                            e = cur.end
                    cur_ind += 1
                lab = CtmInterval(b, e, word, u.name)
                actual_labels.append(lab)
            u.word_labels = actual_labels
            u.phone_labels = [
                x for x in u.phone_labels if x.label != dictionary.optional_silence_phone
            ]

    def construct_output_path(
        self,
        output_directory: Optional[str] = None,
        enforce_lab: bool = False,
    ) -> str:
        """
        Construct the output path for the File

        Parameters
        ----------
        output_directory: str, optional
            Directory to output to, if None, it will overwrite the original file
        enforce_lab: bool
            Flag for whether to enforce generating a lab file over a TextGrid

        Returns
        -------
        str
            Output path
        """
        if enforce_lab:
            extension = ".lab"
        else:
            extension = ".TextGrid"
        if output_directory is None:
            if self.text_path is None:
                return os.path.splitext(self.wav_path)[0] + extension
            return self.text_path
        if self.relative_path:
            relative = os.path.join(output_directory, self.relative_path)
        else:
            relative = output_directory
        tg_path = os.path.join(relative, self._name + extension)
        os.makedirs(os.path.dirname(tg_path), exist_ok=True)
        return tg_path

    def load_text(
        self,
        root_speaker: Optional[Speaker] = None,
        sanitize_function: Optional[MultispeakerSanitizationFunction] = None,
    ) -> None:
        """
        Load the transcription text from the text_file of the object

        Parameters
        ----------
        root_speaker: :class:`~montreal_forced_aligner.corpus.classes.Speaker`, optional
            Speaker derived from the root directory, ignored for TextGrids
        sanitize_function: :class:`~montreal_forced_aligner.dictionary.mixins.SanitizeFunction`, optional
            Function to sanitize words and strip punctuation
        """
        if self.text_type == TextFileType.LAB:
            try:
                text = load_text(self.text_path)
            except UnicodeDecodeError:
                raise TextParseError(self.text_path)
            utterance = Utterance(speaker=root_speaker, file=self, text=text)
            utterance.parse_transcription(sanitize_function)
            self.add_utterance(utterance)
        elif self.text_type == TextFileType.TEXTGRID:
            try:
                tg = textgrid.openTextgrid(self.text_path, includeEmptyIntervals=False)
            except Exception:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                raise TextGridParseError(
                    self.text_path,
                    "\n".join(traceback.format_exception(exc_type, exc_value, exc_traceback)),
                )

            num_tiers = len(tg.tierNameList)
            if num_tiers == 0:
                raise TextGridParseError(self.text_path, "Number of tiers parsed was zero")
            if self.num_channels > 2:
                raise (Exception("More than two channels"))
            for tier_name in tg.tierNameList:
                ti = tg.tierDict[tier_name]
                if tier_name.lower() == "notes":
                    continue
                if not isinstance(ti, textgrid.IntervalTier):
                    continue
                if not root_speaker:
                    speaker_name = tier_name.strip()
                    speaker = Speaker(speaker_name)
                    self.add_speaker(speaker)
                else:
                    speaker = root_speaker
                for begin, end, text in ti.entryList:
                    text = text.lower().strip()
                    if not text:
                        continue
                    begin, end = round(begin, 4), round(end, 4)
                    end = min(end, self.duration)
                    utt = Utterance(speaker=speaker, file=self, begin=begin, end=end, text=text)
                    utt.parse_transcription(sanitize_function)
                    if not utt.text:
                        continue
                    self.add_utterance(utt)
        else:
            utterance = Utterance(speaker=root_speaker, file=self)
            self.add_utterance(utterance)

    def add_speaker(self, speaker: Speaker) -> None:
        """
        Add a speaker to a file

        Parameters
        ----------
        speaker: :class:`~montreal_forced_aligner.corpus.classes.Speaker`
            Speaker to add
        """
        if speaker not in self.speaker_ordering:
            self.speaker_ordering.append(speaker)

    def add_utterance(self, utterance: Utterance) -> None:
        """
        Add an utterance to a file

        Parameters
        ----------
        utterance: :class:`~montreal_forced_aligner.corpus.classes.Utterance`
            Utterance to add
        """
        self.utterances.add_utterance(utterance)
        self.add_speaker(utterance.speaker)

    def delete_utterance(self, utterance: Utterance) -> None:
        """
        Delete an utterance from the file

        Parameters
        ----------
        utterance: :class:`~montreal_forced_aligner.corpus.classes.Utterance`
            Utterance to remove
        """
        identifier = utterance.name
        del self.utterances[identifier]

    def load_info(self) -> None:
        """
        Load sound file info if it hasn't been already
        """
        if self.wav_path is not None:
            self.wav_info = get_wav_info(self.wav_path)

    @property
    def duration(self) -> float:
        """Get the duration of the sound file"""
        if self.wav_path is None:
            return 0
        if not self.wav_info:
            self.load_info()
        return self.wav_info.duration

    @property
    def num_channels(self) -> int:
        """Get the number of channels of the sound file"""
        if self.wav_path is None:
            return 0
        if not self.wav_info:
            self.load_info()
        return self.wav_info.num_channels

    @property
    def num_utterances(self) -> int:
        """Get the number of utterances for the sound file"""
        return len(self.utterances)

    @property
    def num_speakers(self) -> int:
        """Get the number of speakers in the sound file"""
        return len(self.speaker_ordering)

    @property
    def sample_rate(self) -> int:
        """Get the sample rate of the sound file"""
        if self.wav_path is None:
            return 0
        if not self.wav_info:
            self.load_info()
        return self.wav_info.sample_rate

    @property
    def format(self) -> str:
        """Get the sound file format"""
        if not self.wav_info:
            self.load_info()
        return self.wav_info.format

    @property
    def sox_string(self) -> str:
        """String used for converting sound file via SoX within Kaldi"""
        if not self.wav_info:
            self.load_info()
        return self.wav_info.sox_string

    def load_wav_data(self) -> None:
        """
        Load the sound file into memory as a numpy array
        """
        self.waveform, _ = librosa.load(self.wav_path, sr=None, mono=False)

    def normalized_waveform(
        self, begin: float = 0, end: Optional[float] = None
    ) -> Tuple[np.array, np.array]:
        if self.waveform is None:
            self.load_wav_data()
        if end is None or end > self.duration:
            end = self.duration

        begin_sample = int(begin * self.sample_rate)
        end_sample = int(end * self.sample_rate)
        if len(self.waveform.shape) > 1 and self.waveform.shape[0] == 2:
            y = self.waveform[:, begin_sample:end_sample] / np.max(
                np.abs(self.waveform[:, begin_sample:end_sample]), axis=0
            )
            y[np.isnan(y)] = 0
        else:
            y = self.waveform[begin_sample:end_sample] / np.max(
                np.abs(self.waveform[begin_sample:end_sample]), axis=0
            )
        x = np.arange(start=begin_sample, stop=end_sample) / self.sample_rate
        return x, y

    def for_wav_scp(self) -> str:
        """
        Generate the string to use in feature generation

        Returns
        -------
        str
            SoX string if necessary, the sound file path otherwise
        """
        if self.sox_string:
            return self.sox_string
        return self.wav_path
montreal_forced_aligner/dictionary/multispeaker.py
class MultispeakerDictionaryMixin(TemporaryDictionaryMixin, metaclass=abc.ABCMeta):
    """
    Mixin class containing information about a pronunciation dictionary with different dictionaries per speaker

    Parameters
    ----------
    dictionary_path : str
        Dictionary path
    kwargs : kwargs
        Extra parameters to passed to parent classes (see below)

    See Also
    --------
    :class:`~montreal_forced_aligner.dictionary.mixins.DictionaryMixin`
        For dictionary parsing parameters
    :class:`~montreal_forced_aligner.abc.TemporaryDirectoryMixin`
        For temporary directory parameters


    Attributes
    ----------
    dictionary_model: :class:`~montreal_forced_aligner.models.DictionaryModel`
        Dictionary model
    speaker_mapping: dict[str, str]
        Mapping of speaker names to dictionary names
    dictionary_mapping: dict[str, :class:`~montreal_forced_aligner.dictionary.pronunciation.PronunciationDictionary`]
        Mapping of dictionary names to pronunciation dictionary
    """

    def __init__(self, dictionary_path: str = None, **kwargs):
        super().__init__(**kwargs)
        self.dictionary_model = DictionaryModel(
            dictionary_path, phone_set_type=self.phone_set_type
        )
        self.speaker_mapping = {}
        self.dictionary_mapping: Dict[str, PronunciationDictionary] = {}

    @property
    def sanitize_function(self) -> MultispeakerSanitizationFunction:
        """Sanitization function for the dictionary"""
        sanitize_function = SanitizeFunction(
            self.punctuation,
            self.clitic_markers,
            self.compound_markers,
            self.brackets,
            self.ignore_case,
        )
        split_functions = {}
        for dictionary_name, dictionary in self.dictionary_mapping.items():
            split_functions[dictionary_name] = SplitWordsFunction(
                self.clitic_markers,
                self.compound_markers,
                dictionary.clitic_set,
                set(dictionary.actual_words.keys()),
            )
        return MultispeakerSanitizationFunction(
            self.speaker_mapping, sanitize_function, split_functions
        )

    def dictionary_setup(self):
        """Setup the dictionary for processing"""
        auto_set = {PhoneSetType.AUTO, PhoneSetType.UNKNOWN, "AUTO", "UNKNOWN"}
        if not isinstance(self.phone_set_type, PhoneSetType):
            self.phone_set_type = PhoneSetType[self.phone_set_type]

        options = self.dictionary_options
        pretrained = False
        if self.non_silence_phones:
            pretrained = True

        for speaker, dictionary in self.dictionary_model.load_dictionary_paths().items():
            self.speaker_mapping[speaker] = dictionary.name
            if dictionary.name not in self.dictionary_mapping:
                if not pretrained:
                    options["non_silence_phones"] = set()
                self.dictionary_mapping[dictionary.name] = PronunciationDictionary(
                    dictionary_path=dictionary.path,
                    temporary_directory=self.dictionary_output_directory,
                    root_dictionary=self,
                    **options,
                )
                if self.phone_set_type not in auto_set:
                    if (
                        self.phone_set_type
                        != self.dictionary_mapping[dictionary.name].phone_set_type
                    ):
                        raise DictionaryError(
                            f"Mismatch found in phone sets: {self.phone_set_type} vs {self.dictionary_mapping[dictionary.name].phone_set_type}"
                        )
                else:
                    self.phone_set_type = self.dictionary_mapping[dictionary.name].phone_set_type

                self.excluded_phones.update(
                    self.dictionary_mapping[dictionary.name].excluded_phones
                )
                self.excluded_pronunciation_count += self.dictionary_mapping[
                    dictionary.name
                ].excluded_pronunciation_count
        for dictionary in self.dictionary_mapping.values():
            self.non_silence_phones.update(dictionary.non_silence_phones)
        for dictionary in self.dictionary_mapping.values():
            dictionary.non_silence_phones = self.non_silence_phones

    @property
    def name(self) -> str:
        """Name of the dictionary"""
        return self.dictionary_model.name

    def calculate_oovs_found(self) -> None:
        """Sum the counts of oovs found in pronunciation dictionaries"""
        for dictionary in self.dictionary_mapping.values():
            self.oovs_found.update(dictionary.oovs_found)
        self.save_oovs_found(self.output_directory)

    @property
    def default_dictionary(self) -> PronunciationDictionary:
        """Default pronunciation dictionary"""
        return self.get_dictionary("default")

    def get_dictionary_name(self, speaker: Union[str, Speaker]) -> str:
        """
        Get the dictionary name for a given speaker

        Parameters
        ----------
        speaker: Union[Speaker, str]
            Speaker to look up

        Returns
        -------
        str
            Dictionary name for the speaker
        """
        if not isinstance(speaker, str):
            speaker = speaker.name
        if speaker not in self.speaker_mapping:
            return self.speaker_mapping["default"]
        return self.speaker_mapping[speaker]

    def get_dictionary(self, speaker: Union[Speaker, str]) -> PronunciationDictionary:
        """
        Get a dictionary for a given speaker

        Parameters
        ----------
        speaker: Union[Speaker, str]
            Speaker to look up

        Returns
        -------
        :class:`~montreal_forced_aligner.dictionary.PronunciationDictionary`
            Pronunciation dictionary for the speaker
        """
        return self.dictionary_mapping[self.get_dictionary_name(speaker)]

    def write_lexicon_information(self, write_disambiguation: Optional[bool] = False) -> None:
        """
        Write all child dictionaries to the temporary directory

        Parameters
        ----------
        write_disambiguation: bool, optional
            Flag to use disambiguation symbols in the output
        """
        os.makedirs(self.phones_dir, exist_ok=True)
        for d in self.dictionary_mapping.values():
            d.generate_mappings()
            if d.max_disambiguation_symbol > self.max_disambiguation_symbol:
                self.max_disambiguation_symbol = d.max_disambiguation_symbol
        self._write_word_boundaries()
        self._write_phone_sets()
        self._write_phone_symbol_table()
        self._write_disambig()
        self._write_topo()
        self._write_extra_questions()
        for d in self.dictionary_mapping.values():
            d.write(write_disambiguation, debug=getattr(self, "debug", False))

    def set_lexicon_word_set(self, word_set: Collection[str]) -> None:
        """
        Limit output to a subset of overall words

        Parameters
        ----------
        word_set: Collection[str]
            Word set to limit generated files to
        """
        for d in self.dictionary_mapping.values():
            d.set_lexicon_word_set(word_set)

    @property
    def output_paths(self) -> Dict[str, str]:
        """
        Mapping of output directory for child directories
        """
        return {d.name: d.dictionary_output_directory for d in self.dictionary_mapping.values()}
montreal_forced_aligner/corpus/features.py
class MfccArguments(NamedTuple):
    """
    Arguments for :class:`~montreal_forced_aligner.corpus.features.MfccFunction`
    """

    log_path: str
    wav_path: str
    segment_path: str
    feats_scp_path: str
    mfcc_options: MetaDict
montreal_forced_aligner/corpus/base.py
class CorpusMixin(MfaWorker, TemporaryDirectoryMixin, metaclass=ABCMeta):
    """
    Mixin class for processing corpora

    Notes
    -----
    Using characters in files to specify speakers is generally finicky and leads to errors, so I would not
    recommend using it.  Additionally, consider it deprecated and could be removed in future versions

    Parameters
    ----------
    corpus_directory: str
        Path to corpus
    speaker_characters: int or str, optional
        Number of characters in the file name to specify the speaker
    ignore_speakers: bool
        Flag for whether to discard any parsed speaker information during top-level worker's processing

    See Also
    --------
    :class:`~montreal_forced_aligner.abc.MfaWorker`
        For MFA processing parameters
    :class:`~montreal_forced_aligner.abc.TemporaryDirectoryMixin`
        For temporary directory parameters

    Attributes
    ----------
    speakers: :class:`~montreal_forced_aligner.corpus.classes.SpeakerCollection`
        Dictionary of speakers in the corpus
    files: :class:`~montreal_forced_aligner.corpus.classes.FileCollection`
        Dictionary of files in the corpus
    utterances: :class:`~montreal_forced_aligner.corpus.classes.UtteranceCollection`
        Dictionary of utterances in the corpus
    jobs: list[Job]
        List of jobs for processing the corpus and splitting speakers
    word_counts: Counter
        Counts of words in the corpus
    stopped: Stopped
        Stop check for loading the corpus
    decode_error_files: list[str]
        List of text files that could not be loaded with utf8
    textgrid_read_errors: list[str]
        List of TextGrid files that had an error in loading
    """

    def __init__(
        self,
        corpus_directory: str,
        speaker_characters: Union[int, str] = 0,
        ignore_speakers: bool = False,
        **kwargs,
    ):
        if not os.path.exists(corpus_directory):
            raise CorpusError(f"The directory '{corpus_directory}' does not exist.")
        if not os.path.isdir(corpus_directory):
            raise CorpusError(
                f"The specified path for the corpus ({corpus_directory}) is not a directory."
            )
        self.speakers = SpeakerCollection()
        self.files = FileCollection()
        self.utterances = UtteranceCollection()
        self.corpus_directory = corpus_directory
        self.speaker_characters = speaker_characters
        self.ignore_speakers = ignore_speakers
        self.word_counts = Counter()
        self.stopped = Stopped()
        self.decode_error_files = []
        self.textgrid_read_errors = {}
        self.jobs: List[Job] = []
        super().__init__(**kwargs)

    def _initialize_from_json(self, data):
        pass

    @property
    def corpus_meta(self):
        return {}

    @property
    def features_directory(self) -> str:
        """Feature directory of the corpus"""
        return os.path.join(self.corpus_output_directory, "features")

    @property
    def features_log_directory(self) -> str:
        """Feature log directory"""
        return os.path.join(self.split_directory, "log")

    @property
    def split_directory(self) -> str:
        """Directory used to store information split by job"""
        return os.path.join(self.corpus_output_directory, f"split{self.num_jobs}")

    def write_corpus_information(self) -> None:
        """
        Output information to the temporary directory for later loading
        """
        os.makedirs(self.split_directory, exist_ok=True)
        self._write_speakers()
        self._write_files()
        self._write_utterances()
        self._write_spk2utt()
        self._write_corpus_info()

    def _write_spk2utt(self):
        """Write spk2utt scp file for Kaldi"""
        data = {
            speaker.name: sorted(u.name for u in speaker.utterances) for speaker in self.speakers
        }
        output_mapping(data, os.path.join(self.corpus_output_directory, "spk2utt.scp"))

    def write_utt2spk(self):
        """Write utt2spk scp file for Kaldi"""
        data = {u.name: u.speaker.name for u in self.utterances}
        output_mapping(data, os.path.join(self.corpus_output_directory, "utt2spk.scp"))

    def _write_corpus_info(self):
        """Write speaker information for speeding up future runs"""
        with open(
            os.path.join(self.corpus_output_directory, "corpus.json"), "w", encoding="utf8"
        ) as f:
            json.dump(self.corpus_meta, f)

    def _write_speakers(self):
        """Write speaker information for speeding up future runs"""
        with open(
            os.path.join(self.corpus_output_directory, "speakers.jsonl"), "w", encoding="utf8"
        ) as f:
            writer = jsonlines.Writer(f, dumps=jsonl_encoder)
            for speaker in self.speakers:
                writer.write(speaker.meta)

    def _write_files(self):
        """Write file information for speeding up future runs"""
        with open(
            os.path.join(self.corpus_output_directory, "files.jsonl"), "w", encoding="utf8"
        ) as f:
            writer = jsonlines.Writer(f, dumps=jsonl_encoder)
            for file in self.files:
                writer.write(file.meta)

    def _write_utterances(self):
        """Write utterance information for speeding up future runs"""
        with open(
            os.path.join(self.corpus_output_directory, "utterances.jsonl"), "w", encoding="utf8"
        ) as f:
            writer = jsonlines.Writer(f, dumps=jsonl_encoder)
            for utterance in self.utterances:
                writer.write(utterance.meta)

    def create_corpus_split(self) -> None:
        """Create split directory and output information from Jobs"""
        split_dir = self.split_directory
        os.makedirs(os.path.join(split_dir, "log"), exist_ok=True)
        for job in self.jobs:
            job.output_to_directory(split_dir)

    @property
    def file_speaker_mapping(self) -> Dict[str, List[str]]:
        """Speaker ordering for each file"""
        return {file.name: file.speaker_ordering for file in self.files}

    def get_word_frequency(self) -> Dict[str, float]:
        """
        Calculate the relative word frequency across all the texts in the corpus

        Returns
        -------
        dict[str, float]
            Dictionary of words and their relative frequencies
        """
        word_counts = Counter()
        for u in self.utterances:
            text = u.text
            speaker = u.speaker
            d = speaker.dictionary
            new_text = []
            text = text.split()
            for t in text:

                lookup = d.split_clitics(t)
                if lookup is None:
                    continue
                new_text.extend(x for x in lookup if x != "")
            word_counts.update(new_text)
        return {k: v / sum(word_counts.values()) for k, v in word_counts.items()}

    @property
    def corpus_word_set(self) -> List[str]:
        """Set of words used in the corpus"""
        return sorted(self.word_counts)

    def add_utterance(self, utterance: Utterance) -> None:
        """
        Add an utterance to the corpus

        Parameters
        ----------
        utterance: :class:`~montreal_forced_aligner.corpus.classes.Utterance`
            Utterance to add
        """
        self.utterances.add_utterance(utterance)
        if utterance.speaker not in self.speakers:
            self.speakers.add_speaker(utterance.speaker)
        speaker = self.speakers[utterance.speaker.name]
        speaker.add_utterance(utterance)
        utterance.speaker = speaker
        if utterance.file not in self.files:
            self.files.add_file(utterance.file)
        file = self.files[utterance.file.name]
        utterance.file = file
        file.add_utterance(utterance)
        utterance.file = file

    def delete_utterance(self, utterance: Union[str, Utterance]) -> None:
        """
        Delete an utterance from the corpus

        Parameters
        ----------
        utterance: :class:`~montreal_forced_aligner.corpus.classes.Utterance`
            Utterance to delete
        """
        if isinstance(utterance, str):
            utterance = self.utterances[utterance]
        speaker = self.speakers[utterance.speaker.name]
        file = self.files[utterance.file.name]
        speaker.delete_utterance(utterance)
        file.delete_utterance(utterance)
        del self.utterances[utterance.name]

    def initialize_jobs(self) -> None:
        """
        Initialize the corpus's Jobs
        """
        self.log_info("Setting up training data...")
        if len(self.speakers) < self.num_jobs:
            self.num_jobs = len(self.speakers)
        self.jobs = [Job(i) for i in range(self.num_jobs)]
        job_ind = 0
        for s in sorted(self.speakers):
            self.jobs[job_ind].add_speaker(s)
            job_ind += 1
            if job_ind == self.num_jobs:
                job_ind = 0

    def add_file(self, file: File) -> None:
        """
        Add a file to the corpus

        Parameters
        ----------
        file: :class:`~montreal_forced_aligner.corpus.classes.File`
            File to be added
        """
        self.files.add_file(file)
        for i, speaker in enumerate(file.speaker_ordering):
            if speaker.name not in self.speakers:
                self.speakers.add_speaker(speaker)
            else:
                file.speaker_ordering[i] = self.speakers[speaker.name]
        for u in file.utterances:
            self.add_utterance(u)
            if u.text:
                self.word_counts.update(u.text.split())
            if u.normalized_text:
                self.word_counts.update(u.normalized_text)

    @property
    def data_source_identifier(self) -> str:
        """Corpus name"""
        return os.path.basename(self.corpus_directory)

    def create_subset(self, subset: int) -> None:
        """
        Create a subset of utterances to use for training

        Parameters
        ----------
        subset: int
            Number of utterances to include in subset
        """
        self.log_debug(f"Creating subset_{subset} directory...")
        subset_directory = os.path.join(self.corpus_output_directory, f"subset_{subset}")

        larger_subset_num = subset * 10
        if larger_subset_num < len(self.utterances):
            # Get all shorter utterances that are not one word long
            utts = sorted(
                (utt for utt in self.utterances if utt.text and " " in utt.text),
                key=lambda x: x.duration,
            )
            larger_subset = utts[:larger_subset_num]
        else:
            larger_subset = sorted(self.utterances)
        random.seed(1234)  # make it deterministic sampling
        subset_utts = UtteranceCollection()
        subset_utts.update(random.sample(larger_subset, subset))
        log_dir = os.path.join(subset_directory, "log")
        os.makedirs(log_dir, exist_ok=True)

        for j in self.jobs:
            j.set_subset(subset_utts)
            j.output_to_directory(subset_directory)

    @property
    def num_utterances(self) -> int:
        """Number of utterances in the corpus"""
        return len(self.utterances)

    @property
    def num_speakers(self) -> int:
        """Number of speakers in the corpus"""
        return len(self.speakers)

    def subset_directory(self, subset: Optional[int]) -> str:
        """
        Construct a subset directory for the corpus

        Parameters
        ----------
        subset: int, optional
            Number of utterances to include, if larger than the total number of utterance or not specified, the
            split_directory is returned

        Returns
        -------
        str
            Path to subset directory
        """
        if subset is None or subset >= len(self.utterances) or subset <= 0:
            for j in self.jobs:
                j.set_subset(None)
            return self.split_directory
        directory = os.path.join(self.corpus_output_directory, f"subset_{subset}")
        if not os.path.exists(directory):
            self.create_subset(subset)
        return directory

    def _load_corpus(self) -> None:
        """
        Load the corpus
        """
        self.log_info("Setting up corpus information...")
        loaded = self._load_corpus_from_temp()
        if not loaded:
            self.log_debug("Could not load from temp")
            self.log_info("Loading corpus from source files...")
            if self.use_mp:

                self._load_corpus_from_source_mp()
            else:
                self._load_corpus_from_source()
        else:
            self.log_debug("Successfully loaded from temporary files")
        if not self.files:
            raise CorpusError(
                "There were no files found for this corpus. Please validate the corpus."
            )
        num_speakers = len(self.speakers)
        if not num_speakers:
            raise CorpusError(
                "There were no sound files found of the appropriate format. Please double check the corpus path "
                "and/or run the validation utility (mfa validate)."
            )
        average_utterances = sum(len(x.utterances) for x in self.speakers) / num_speakers
        self.log_info(
            f"Number of speakers in corpus: {num_speakers}, "
            f"average number of utterances per speaker: {average_utterances}"
        )

    def _load_corpus_from_temp(self) -> bool:
        """
        Load a corpus from saved data in the temporary directory

        Returns
        -------
        bool
            Whether loading from temporary files was successful
        """
        begin_time = time.time()
        if not os.path.exists(self.corpus_output_directory):
            return False
        for f in os.listdir(self.corpus_output_directory):
            if f.startswith("split"):
                old_num_jobs = int(f.replace("split", ""))
                if old_num_jobs != self.num_jobs:
                    self.log_info(
                        f"Found old run with {old_num_jobs} rather than the current {self.num_jobs}, "
                        f"setting to {old_num_jobs}.  If you would like to use {self.num_jobs}, re-run the command "
                        f"with --clean."
                    )
                    self.num_jobs = old_num_jobs
        format = "jsonl"
        corpus_path = os.path.join(self.corpus_output_directory, "corpus.json")
        speakers_path = os.path.join(self.corpus_output_directory, "speakers.jsonl")
        files_path = os.path.join(self.corpus_output_directory, "files.jsonl")
        utterances_path = os.path.join(self.corpus_output_directory, "utterances.jsonl")
        if not os.path.exists(speakers_path):
            format = "yaml"
            speakers_path = os.path.join(self.corpus_output_directory, "speakers.yaml")
            files_path = os.path.join(self.corpus_output_directory, "files.yaml")
            utterances_path = os.path.join(self.corpus_output_directory, "utterances.yaml")

        if not os.path.exists(speakers_path):
            self.log_debug(f"Could not find {speakers_path}, cannot load from temp")
            return False
        if not os.path.exists(files_path):
            self.log_debug(f"Could not find {files_path}, cannot load from temp")
            return False
        if not os.path.exists(utterances_path):
            self.log_debug(f"Could not find {utterances_path}, cannot load from temp")
            return False
        self.log_debug("Loading from temporary files...")

        if os.path.exists(corpus_path):
            with open(corpus_path, "r", encoding="utf8") as f:
                data = json.load(f)
                self._initialize_from_json(data)

        with open(speakers_path, "r", encoding="utf8") as f:
            if format == "jsonl":
                speaker_data = jsonlines.Reader(f)
            else:
                speaker_data = yaml.safe_load(f)

            for entry in speaker_data:
                self.speakers.add_speaker(Speaker(entry["name"]))
                self.speakers[entry["name"]].cmvn = entry["cmvn"]

        with open(files_path, "r", encoding="utf8") as f:
            if format == "jsonl":
                files_data = jsonlines.Reader(f)
            else:
                files_data = yaml.safe_load(f)
            for entry in files_data:
                file = File(
                    name=entry["name"],
                    wav_path=entry["wav_path"],
                    text_path=entry["text_path"],
                    relative_path=entry["relative_path"],
                )
                self.files.add_file(file)
                self.files[file.name].speaker_ordering = [
                    self.speakers[x] for x in entry["speaker_ordering"]
                ]
                self.files[file.name].wav_info = SoundFileInformation(**entry["wav_info"])

        with open(utterances_path, "r", encoding="utf8") as f:
            if format == "jsonl":
                utterances_data = jsonlines.Reader(f)
            else:
                utterances_data = yaml.safe_load(f)
            for entry in utterances_data:
                s = self.speakers[entry["speaker"]]
                f = self.files[entry["file"]]
                u = Utterance(
                    s,
                    f,
                    begin=entry["begin"],
                    end=entry["end"],
                    channel=entry["channel"],
                    text=entry["text"],
                )
                u.oovs = set(entry["oovs"])
                u.normalized_text = entry["normalized_text"]
                if u.text:
                    self.word_counts.update(u.text.split())
                if u.normalized_text:
                    self.word_counts.update(u.normalized_text)
                u.word_error_rate = entry.get("word_error_rate", None)
                u.character_error_rate = entry.get("character_error_rate", None)
                u.transcription_text = entry.get("transcription_text", None)
                u.phone_error_rate = entry.get("phone_error_rate", None)
                u.alignment_score = entry.get("alignment_score", None)
                u.alignment_log_likelihood = entry.get("alignment_log_likelihood", None)
                u.reference_phone_labels = [
                    CtmInterval(**x) for x in entry.get("reference_phone_labels", [])
                ]

                phone_labels = entry.get("phone_labels", None)
                if phone_labels:
                    u.phone_labels = [CtmInterval(**x) for x in phone_labels]
                word_labels = entry.get("word_labels", None)
                if word_labels:
                    u.word_labels = [CtmInterval(**x) for x in word_labels]
                u.features = entry.get("features", None)
                u.ignored = entry.get("ignored", False)
                self.add_utterance(u)

        self.log_debug(
            f"Loaded from corpus_data temp directory in {time.time() - begin_time} seconds"
        )
        return True

    @property
    def base_data_directory(self) -> str:
        """Corpus data directory"""
        return self.corpus_output_directory

    @property
    def data_directory(self) -> str:
        """Corpus data directory"""
        return self.split_directory

    @abstractmethod
    def _load_corpus_from_source_mp(self) -> None:
        """Abstract method for loading a corpus with multiprocessing"""
        ...

    @abstractmethod
    def _load_corpus_from_source(self) -> None:
        """Abstract method for loading a corpus without multiprocessing"""
        ...
montreal_forced_aligner/corpus/features.py
class CalcFmllrArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.corpus.features.CalcFmllrFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    ali_paths: Dict[str, str]
    ali_model_path: str
    model_path: str
    spk2utt_paths: Dict[str, str]
    trans_paths: Dict[str, str]
    fmllr_options: MetaDict
montreal_forced_aligner/corpus/features.py
class CalcFmllrFunction(KaldiFunction):
    """
    Multiprocessing function for calculating fMLLR transforms

    See Also
    --------
    :meth:`.AcousticCorpusMixin.calc_fmllr`
        Main function that calls this function in parallel
    :meth:`.AcousticCorpusMixin.calc_fmllr_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`gmm-est-fmllr`
        Relevant Kaldi binary
    :kaldi_src:`gmm-est-fmllr-gpost`
        Relevant Kaldi binary
    :kaldi_src:`gmm-post-to-gpost`
        Relevant Kaldi binary
    :kaldi_src:`ali-to-post`
        Relevant Kaldi binary
    :kaldi_src:`weight-silence-post`
        Relevant Kaldi binary
    :kaldi_src:`compose-transforms`
        Relevant Kaldi binary
    :kaldi_src:`transform-feats`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.corpus.features.CalcFmllrArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(r"^LOG.*For speaker (?P<speaker>.*),.*$")

    def __init__(self, args: CalcFmllrArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.ali_paths = args.ali_paths
        self.ali_model_path = args.ali_model_path
        self.model_path = args.model_path
        self.spk2utt_paths = args.spk2utt_paths
        self.trans_paths = args.trans_paths
        self.fmllr_options = args.fmllr_options

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                feature_string = self.feature_strings[dict_name]
                ali_path = self.ali_paths[dict_name]
                spk2utt_path = self.spk2utt_paths[dict_name]
                trans_path = self.trans_paths[dict_name]
                initial = True
                if os.path.exists(trans_path):
                    initial = False
                post_proc = subprocess.Popen(
                    [thirdparty_binary("ali-to-post"), f"ark:{ali_path}", "ark:-"],
                    stderr=log_file,
                    stdout=subprocess.PIPE,
                    env=os.environ,
                )

                weight_proc = subprocess.Popen(
                    [
                        thirdparty_binary("weight-silence-post"),
                        "0.0",
                        self.fmllr_options["silence_csl"],
                        self.ali_model_path,
                        "ark:-",
                        "ark:-",
                    ],
                    stderr=log_file,
                    stdin=post_proc.stdout,
                    stdout=subprocess.PIPE,
                    env=os.environ,
                )

                temp_trans_path = trans_path + ".tmp"
                if self.ali_model_path != self.model_path:
                    post_gpost_proc = subprocess.Popen(
                        [
                            thirdparty_binary("gmm-post-to-gpost"),
                            self.ali_model_path,
                            feature_string,
                            "ark:-",
                            "ark:-",
                        ],
                        stderr=log_file,
                        stdin=weight_proc.stdout,
                        stdout=subprocess.PIPE,
                        env=os.environ,
                    )
                    est_proc = subprocess.Popen(
                        [
                            thirdparty_binary("gmm-est-fmllr-gpost"),
                            "--verbose=4",
                            f"--fmllr-update-type={self.fmllr_options['fmllr_update_type']}",
                            f"--spk2utt=ark:{spk2utt_path}",
                            self.model_path,
                            feature_string,
                            "ark,s,cs:-",
                            f"ark:{temp_trans_path}",
                        ],
                        stderr=subprocess.PIPE,
                        encoding="utf8",
                        stdin=post_gpost_proc.stdout,
                        env=os.environ,
                    )

                else:

                    if not initial:
                        temp_composed_trans_path = trans_path + ".cmp.tmp"
                        est_proc = subprocess.Popen(
                            [
                                thirdparty_binary("gmm-est-fmllr"),
                                "--verbose=4",
                                f"--fmllr-update-type={self.fmllr_options['fmllr_update_type']}",
                                f"--spk2utt=ark:{spk2utt_path}",
                                self.model_path,
                                feature_string,
                                "ark:-",
                                f"ark:{temp_trans_path}",
                            ],
                            stderr=subprocess.PIPE,
                            encoding="utf8",
                            stdin=weight_proc.stdout,
                            stdout=subprocess.PIPE,
                            env=os.environ,
                        )
                    else:
                        est_proc = subprocess.Popen(
                            [
                                thirdparty_binary("gmm-est-fmllr"),
                                "--verbose=4",
                                f"--fmllr-update-type={self.fmllr_options['fmllr_update_type']}",
                                f"--spk2utt=ark:{spk2utt_path}",
                                self.model_path,
                                feature_string,
                                "ark,s,cs:-",
                                f"ark:{trans_path}",
                            ],
                            stderr=subprocess.PIPE,
                            encoding="utf8",
                            stdin=weight_proc.stdout,
                            env=os.environ,
                        )

                for line in est_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield m.group("speaker")
                if not initial:
                    compose_proc = subprocess.Popen(
                        [
                            thirdparty_binary("compose-transforms"),
                            "--b-is-affine=true",
                            f"ark:{temp_trans_path}",
                            f"ark:{trans_path}",
                            f"ark:{temp_composed_trans_path}",
                        ],
                        stderr=log_file,
                        env=os.environ,
                    )
                    compose_proc.communicate()

                    os.remove(trans_path)
                    os.remove(temp_trans_path)
                    os.rename(temp_composed_trans_path, trans_path)
montreal_forced_aligner/corpus/features.py
class MfccFunction(KaldiFunction):
    """
    Multiprocessing function for generating MFCC features

    See Also
    --------
    :meth:`.AcousticCorpusMixin.mfcc`
        Main function that calls this function in parallel
    :meth:`.AcousticCorpusMixin.mfcc_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`compute-mfcc-feats`
        Relevant Kaldi binary
    :kaldi_src:`extract-segments`
        Relevant Kaldi binary
    :kaldi_src:`copy-feats`
        Relevant Kaldi binary
    :kaldi_src:`feat-to-len`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.corpus.features.MfccArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(r"^LOG.* Copied (?P<num_utterances>\d+) feature matrices.")

    def __init__(self, args: MfccArguments):
        self.log_path = args.log_path
        self.wav_path = args.wav_path
        self.segment_path = args.segment_path
        self.feats_scp_path = args.feats_scp_path
        self.mfcc_options = args.mfcc_options

    def run(self):
        """Run the function"""
        with open(self.log_path, "w") as log_file:
            mfcc_base_command = [thirdparty_binary("compute-mfcc-feats"), "--verbose=2"]
            raw_ark_path = self.feats_scp_path.replace(".scp", ".ark")
            for k, v in self.mfcc_options.items():
                mfcc_base_command.append(f"--{k.replace('_', '-')}={make_safe(v)}")
            if os.path.exists(self.segment_path):
                mfcc_base_command += ["ark:-", "ark:-"]
                seg_proc = subprocess.Popen(
                    [
                        thirdparty_binary("extract-segments"),
                        f"scp,p:{self.wav_path}",
                        self.segment_path,
                        "ark:-",
                    ],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                comp_proc = subprocess.Popen(
                    mfcc_base_command,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    stdin=seg_proc.stdout,
                    env=os.environ,
                )
            else:
                mfcc_base_command += [f"scp,p:{self.wav_path}", "ark:-"]
                comp_proc = subprocess.Popen(
                    mfcc_base_command, stdout=subprocess.PIPE, stderr=log_file, env=os.environ
                )
            copy_proc = subprocess.Popen(
                [
                    thirdparty_binary("copy-feats"),
                    "--verbose=2",
                    "--compress=true",
                    "ark:-",
                    f"ark,scp:{raw_ark_path},{self.feats_scp_path}",
                ],
                stdin=comp_proc.stdout,
                stderr=subprocess.PIPE,
                env=os.environ,
                encoding="utf8",
            )
            for line in copy_proc.stderr:
                m = self.progress_pattern.match(line.strip())
                if m:
                    yield int(m.group("num_utterances"))
montreal_forced_aligner/utils.py
class KaldiProcessWorker(mp.Process):
    """
    Multiprocessing function work

    Parameters
    ----------
    job_name: int
        Integer number of job
    job_q: :class:`~multiprocessing.Queue`
        Job queue to pull arguments from
    function: KaldiFunction
        Multiprocessing function to call on arguments from job_q
    return_dict: dict
        Dictionary for collecting errors
    stopped: :class:`~montreal_forced_aligner.utils.Stopped`
        Stop check
    return_info: dict[int, Any], optional
        Optional dictionary to fill if the function should return information to main thread
    """

    def __init__(
        self,
        job_name: int,
        return_q: mp.Queue,
        function: KaldiFunction,
        error_dict: dict,
        stopped: Stopped,
    ):
        mp.Process.__init__(self)
        self.job_name = job_name
        self.function = function
        self.return_q = return_q
        self.error_dict = error_dict
        self.stopped = stopped
        self.finished = Stopped()

    def run(self) -> None:
        """
        Run through the arguments in the queue apply the function to them
        """
        try:
            for result in self.function.run():
                self.return_q.put(result)
        except Exception:
            self.stopped.stop()
            self.error_dict[self.job_name] = Exception(traceback.format_exception(*sys.exc_info()))
        finally:
            self.finished.stop()
montreal_forced_aligner/corpus/features.py
class FeatureConfigMixin:
    """
    Class to store configuration information about MFCC generation

    Attributes
    ----------
    feature_type : str
        Feature type, defaults to "mfcc"
    use_energy : bool
        Flag for whether first coefficient should be used, defaults to False
    frame_shift : int
        number of milliseconds between frames, defaults to 10
    snip_edges : bool
        Flag for enabling Kaldi's snip edges, should be better time precision
    pitch : bool
        Flag for including pitch in features, currently nonfunctional, defaults to False
    low_frequency : int
        Frequency floor
    high_frequency : int
        Frequency ceiling
    sample_frequency : int
        Sampling frequency
    allow_downsample : bool
        Flag for whether to allow downsampling, default is True
    allow_upsample : bool
        Flag for whether to allow upsampling, default is True
    speaker_independent : bool
        Flag for whether features are speaker independent, default is True
    uses_cmvn : bool
        Flag for whether to use CMVN, default is True
    uses_deltas : bool
        Flag for whether to use delta features, default is True
    uses_splices : bool
        Flag for whether to use splices and LDA transformations, default is False
    uses_speaker_adaptation : bool
        Flag for whether to use speaker adaptation, default is False
    fmllr_update_type : str
        Type of fMLLR estimation, defaults to "full"
    silence_weight : float
        Weight of silence in calculating LDA or fMLLR
    splice_left_context : int or None
        Number of frames to splice on the left for calculating LDA
    splice_right_context : int or None
        Number of frames to splice on the right for calculating LDA
    """

    def __init__(
        self,
        feature_type: str = "mfcc",
        use_energy: bool = False,
        frame_shift: int = 10,
        snip_edges: bool = True,
        pitch: bool = False,
        low_frequency: int = 20,
        high_frequency: int = 7800,
        sample_frequency: int = 16000,
        allow_downsample: bool = True,
        allow_upsample: bool = True,
        speaker_independent: bool = True,
        uses_cmvn: bool = True,
        uses_deltas: bool = True,
        uses_splices: bool = False,
        uses_voiced: bool = False,
        uses_speaker_adaptation: bool = False,
        fmllr_update_type: str = "full",
        silence_weight: float = 0.0,
        splice_left_context: int = 3,
        splice_right_context: int = 3,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.feature_type = feature_type
        self.use_energy = use_energy
        self.frame_shift = frame_shift
        self.snip_edges = snip_edges
        self.pitch = pitch
        self.low_frequency = low_frequency
        self.high_frequency = high_frequency
        self.sample_frequency = sample_frequency
        self.allow_downsample = allow_downsample
        self.allow_upsample = allow_upsample
        self.speaker_independent = speaker_independent
        self.uses_cmvn = uses_cmvn
        self.uses_deltas = uses_deltas
        self.uses_splices = uses_splices
        self.uses_voiced = uses_voiced
        self.uses_speaker_adaptation = uses_speaker_adaptation
        self.fmllr_update_type = fmllr_update_type
        self.silence_weight = silence_weight
        self.splice_left_context = splice_left_context
        self.splice_right_context = splice_right_context

    @property
    def vad_options(self) -> MetaDict:
        """Abstract method for VAD options"""
        raise NotImplementedError

    @property
    def alignment_model_path(self) -> str:  # needed for fmllr
        """Abstract method for alignment model path"""
        raise NotImplementedError

    @property
    def model_path(self) -> str:  # needed for fmllr
        """Abstract method for model path"""
        raise NotImplementedError

    @property
    @abstractmethod
    def working_directory(self) -> str:
        """Abstract method for working directory"""
        ...

    @property
    @abstractmethod
    def corpus_output_directory(self) -> str:
        """Abstract method for working directory of corpus"""
        ...

    @property
    @abstractmethod
    def data_directory(self) -> str:
        """Abstract method for corpus data directory"""
        ...

    @property
    def feature_options(self) -> MetaDict:
        """Parameters for feature generation"""
        options = {
            "type": self.feature_type,
            "use_energy": self.use_energy,
            "frame_shift": self.frame_shift,
            "snip_edges": self.snip_edges,
            "low_frequency": self.low_frequency,
            "high_frequency": self.high_frequency,
            "sample_frequency": self.sample_frequency,
            "allow_downsample": self.allow_downsample,
            "allow_upsample": self.allow_upsample,
            "pitch": self.pitch,
            "uses_cmvn": self.uses_cmvn,
            "uses_deltas": self.uses_deltas,
            "uses_voiced": self.uses_voiced,
            "uses_splices": self.uses_splices,
            "uses_speaker_adaptation": self.uses_speaker_adaptation,
        }
        if self.uses_splices:
            options.update(
                {
                    "splice_left_context": self.splice_left_context,
                    "splice_right_context": self.splice_right_context,
                }
            )
        return options

    @abstractmethod
    def calc_fmllr(self) -> None:
        """Abstract method for calculating fMLLR transforms"""
        ...

    @property
    def fmllr_options(self) -> MetaDict:
        """Options for use in calculating fMLLR transforms"""
        return {
            "fmllr_update_type": self.fmllr_update_type,
            "silence_weight": self.silence_weight,
            "silence_csl": getattr(
                self, "silence_csl", ""
            ),  # If we have silence phones from a dictionary, use them
        }

    @property
    def mfcc_options(self) -> MetaDict:
        """Parameters to use in computing MFCC features."""
        return {
            "use-energy": self.use_energy,
            "frame-shift": self.frame_shift,
            "low-freq": self.low_frequency,
            "high-freq": self.high_frequency,
            "sample-frequency": self.sample_frequency,
            "allow-downsample": self.allow_downsample,
            "allow-upsample": self.allow_upsample,
            "snip-edges": self.snip_edges,
        }
montreal_forced_aligner/corpus/multiprocessing.py
class CorpusProcessWorker(mp.Process):
    """
    Multiprocessing corpus loading worker

    Attributes
    ----------
    job_q: :class:`~multiprocessing.Queue`
        Job queue for files to process
    return_dict: dict
        Dictionary to catch errors
    return_q: :class:`~multiprocessing.Queue`
        Return queue for processed Files
    stopped: :class:`~montreal_forced_aligner.utils.Stopped`
        Stop check for whether corpus loading should exit
    finished_adding: :class:`~montreal_forced_aligner.utils.Stopped`
        Signal that the main thread has stopped adding new files to be processed
    """

    def __init__(
        self,
        name: int,
        job_q: mp.Queue,
        return_dict: dict,
        return_q: mp.Queue,
        stopped: Stopped,
        finished_adding: Stopped,
        speaker_characters: Union[int, str],
        sanitize_function: Optional[MultispeakerSanitizationFunction],
    ):
        mp.Process.__init__(self)
        self.name = str(name)
        self.job_q = job_q
        self.return_dict = return_dict
        self.return_q = return_q
        self.stopped = stopped
        self.finished_adding = finished_adding
        self.finished_processing = Stopped()
        self.sanitize_function = sanitize_function
        self.speaker_characters = speaker_characters

    def run(self) -> None:
        """
        Run the corpus loading job
        """

        while True:
            try:
                file_name, wav_path, text_path, relative_path = self.job_q.get(timeout=1)
            except Empty:
                if self.finished_adding.stop_check():
                    break
                continue
            self.job_q.task_done()
            if self.stopped.stop_check():
                continue
            try:
                file = File.parse_file(
                    file_name,
                    wav_path,
                    text_path,
                    relative_path,
                    self.speaker_characters,
                    self.sanitize_function,
                )

                self.return_q.put(file.multiprocessing_data)
            except TextParseError as e:
                self.return_dict["decode_error_files"].append(e)
            except TextGridParseError as e:
                self.return_dict["textgrid_read_errors"][e.file_name] = e
            except Exception:
                self.stopped.stop()
                self.return_dict["error"] = file_name, Exception(
                    traceback.format_exception(*sys.exc_info())
                )
        self.finished_processing.stop()
        return
montreal_forced_aligner/exceptions.py
class TextGridParseError(CorpusReadError):
    """
    Class capturing TextGrid reading errors

    Parameters
    ----------
    file_name: str
        File name that had the error
    error: str
        Error in TextGrid file
    """

    def __init__(self, file_name: str, error: str):
        super().__init__("")
        self.file_name = file_name
        self.error = error
        self.message_lines.extend(
            [
                f"Reading {self.printer.emphasized_text(file_name)} has the following error:",
                "",
                "",
                self.error,
            ]
        )
montreal_forced_aligner/corpus/features.py
class VadArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.corpus.features.ComputeVadFunction`"""

    log_path: str
    feats_scp_path: str
    vad_scp_path: str
    vad_options: MetaDict
montreal_forced_aligner/exceptions.py
class TextParseError(CorpusReadError):
    """
    Class for errors parsing lab and txt files

    Parameters
    ----------
    file_name: str
        File name that had the error
    """

    def __init__(self, file_name: str):
        super().__init__("")
        self.message_lines = [
            f"There was an error decoding {self.printer.error_text(file_name)}, "
            f"maybe try resaving it as utf8?"
        ]


Question: import multiprocessing as mp
import os
import shutil
import subprocess
import sys
import time
import tqdm
from abc import ABCMeta
from queue import Empty
from typing import Dict, List, Optional
from montreal_forced_aligner.abc import MfaWorker, TemporaryDirectoryMixin
from montreal_forced_aligner.corpus.base import CorpusMixin
from montreal_forced_aligner.corpus.classes import File
from montreal_forced_aligner.corpus.features import (
    CalcFmllrArguments,
    CalcFmllrFunction,
    ComputeVadFunction,
    FeatureConfigMixin,
    MfccArguments,
    MfccFunction,
    VadArguments,
)
from montreal_forced_aligner.corpus.helper import find_exts
from montreal_forced_aligner.corpus.multiprocessing import CorpusProcessWorker
from montreal_forced_aligner.data import TextFileType
from montreal_forced_aligner.dictionary.multispeaker import MultispeakerDictionaryMixin
from montreal_forced_aligner.exceptions import TextGridParseError, TextParseError
from montreal_forced_aligner.helper import load_scp
from montreal_forced_aligner.textgrid import parse_aligned_textgrid
from montreal_forced_aligner.utils import KaldiProcessWorker, Stopped, thirdparty_binary
                            "There were some issues with files in the corpus. "
                            "Please look at the log file or run the validator for more information."
                        )
                        self.log_debug(f"{k} showed {len(return_dict[k])} errors:")
                        if k == "textgrid_read_errors":
                            getattr(self, k).update(return_dict[k])
                            for f, e in return_dict[k].items():
                                self.log_debug(f"{f}: {e.error}")
                        else:
                            self.log_debug(", ".join(return_dict[k]))
                            setattr(self, k, return_dict[k])

        except KeyboardInterrupt:
            self.log_info("Detected ctrl-c, please wait a moment while we clean everything up...")
            self.stopped.stop()
            finished_adding.stop()
            job_queue.join()
            self.stopped.set_sigint_source()
            while True:
                try:
                    _ = return_queue.get(timeout=1)
                    if self.stopped.stop_check():
                        continue
                except Empty:
                    for proc in procs:
                        if not proc.finished_processing.stop_check():
                            break
                    else:
                        break
        finally:

            finished_adding.stop()
            job_queue.join()
            for p in procs:
                p.join()
            if self.stopped.stop_check():
                self.log_info(
                    f"Stopped parsing early ({time.process_time() - begin_time} seconds)"
                )
                if self.stopped.source():
                    sys.exit(0)
            else:
                self.log_debug(
                    f"Parsed corpus directory with {self.num_jobs} jobs in {time.process_time() - begin_time} seconds"
                )

    def _load_corpus_from_source(self) -> None:
        """
        Load a corpus without using multiprocessing
        """
        begin_time = time.time()
        sanitize_function = None
        if hasattr(self, "sanitize_function"):
            sanitize_function = self.sanitize_function

        all_sound_files = {}
        use_audio_directory = False
        if self.audio_directory and os.path.exists(self.audio_directory):
            use_audio_directory = True
            for root, _, files in os.walk(self.audio_directory, followlinks=True):
                if self.stopped.stop_check():
                    return
                exts = find_exts(files)
                exts.wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                exts.other_audio_files = {
                    k: os.path.join(root, v) for k, v in exts.other_audio_files.items()
                }
                all_sound_files.update(exts.other_audio_files)
                all_sound_files.update(exts.wav_files)
        self.log_debug(f"Walking through {self.corpus_directory}...")
        for root, _, files in os.walk(self.corpus_directory, followlinks=True):
            exts = find_exts(files)
            relative_path = root.replace(self.corpus_directory, "").lstrip("/").lstrip("\\")
            if self.stopped.stop_check():
                return
            self.log_debug(f"Inside relative root {relative_path}:")
            self.log_debug(f"    Found {len(exts.identifiers)} identifiers")
            self.log_debug(f"    Found {len(exts.wav_files)} .wav files")
            self.log_debug(f"    Found {len(exts.other_audio_files)} other audio files")
            self.log_debug(f"    Found {len(exts.lab_files)} .lab files")
            self.log_debug(f"    Found {len(exts.textgrid_files)} .TextGrid files")
            if not use_audio_directory:
                all_sound_files = {}
                wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                other_audio_files = {
                    k: os.path.join(root, v) for k, v in exts.other_audio_files.items()
                }
                all_sound_files.update(other_audio_files)
                all_sound_files.update(wav_files)
            for file_name in exts.identifiers:

                wav_path = None
                transcription_path = None
                if file_name in all_sound_files:
                    wav_path = all_sound_files[file_name]
                if file_name in exts.lab_files:
                    lab_name = exts.lab_files[file_name]
                    transcription_path = os.path.join(root, lab_name)
                elif file_name in exts.textgrid_files:
                    tg_name = exts.textgrid_files[file_name]
                    transcription_path = os.path.join(root, tg_name)
                if wav_path is None and transcription_path is None:  # Not a file for MFA
                    continue
                if wav_path is None:
                    self.transcriptions_without_wavs.append(transcription_path)
                    continue
                if transcription_path is None:
                    self.no_transcription_files.append(wav_path)
                try:
                    file = File.parse_file(
                        file_name,
                        wav_path,
                        transcription_path,
                        relative_path,
                        self.speaker_characters,
                        sanitize_function,
                    )
                    self.add_file(file)
                except TextParseError as e:
                    self.decode_error_files.append(e)

Answer:                 except TextGridParseError as e: