montreal_forced_aligner/transcription/multiprocessing.py
class ScoreArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.ScoreFunction`"""

    log_path: str
    dictionaries: List[str]
    score_options: MetaDict
    lat_paths: Dict[str, str]
    rescored_lat_paths: Dict[str, str]
    carpa_rescored_lat_paths: Dict[str, str]
    words_paths: Dict[str, str]
    tra_paths: Dict[str, str]
montreal_forced_aligner/exceptions.py
class PlatformError(MFAError):
    """
    Exception class for platform compatibility issues

    Parameters
    ----------
    functionality_name: str
        Functionality not available on the current platform
    """

    def __init__(self, functionality_name):
        super().__init__("")
        self.message_lines = [
            f"Functionality for {self.printer.emphasized_text(functionality_name)} is not available on {self.printer.error_text(sys.platform)}."
        ]
        if sys.platform == "win32":
            self.message_lines.append("")
            self.message_lines.append(
                f" If you'd like to use {self.printer.emphasized_text(functionality_name)} on Windows, please follow the MFA installation "
                f"instructions for the Windows Subsystem for Linux (WSL)."
            )
montreal_forced_aligner/transcription/multiprocessing.py
class FmllrRescoreFunction(KaldiFunction):
    """
    Multiprocessing function to rescore lattices following fMLLR estimation

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe_fmllr`
        Main function that calls this function in parallel
    :meth:`.Transcriber.fmllr_rescore_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`gmm-rescore-lattice`
        Relevant Kaldi binary
    :kaldi_src:`lattice-determinize-pruned`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.FmllrRescoreArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Done (?P<done>\d+) lattices, determinization finished earlier than specified by the beam (or output was empty) on (?P<errors>\d+) of these."
    )

    def __init__(self, args: FmllrRescoreArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.model_path = args.model_path
        self.fmllr_options = args.fmllr_options
        self.tmp_lat_paths = args.tmp_lat_paths
        self.final_lat_paths = args.final_lat_paths

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                feature_string = self.feature_strings[dict_name]
                tmp_lat_path = self.tmp_lat_paths[dict_name]
                final_lat_path = self.final_lat_paths[dict_name]
                rescore_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-rescore-lattice"),
                        self.model_path,
                        f"ark:{tmp_lat_path}",
                        feature_string,
                        "ark:-",
                    ],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                determinize_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-determinize-pruned"),
                        f"--acoustic-scale={self.fmllr_options['acoustic_scale']}",
                        f"--beam={self.fmllr_options['lattice_beam']}",
                        "ark:-",
                        f"ark:{final_lat_path}",
                    ],
                    stdin=rescore_proc.stdout,
                    stderr=subprocess.PIPE,
                    encoding="utf8",
                    env=os.environ,
                )
                for line in determinize_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield int(m.group("done")), int(m.group("errors"))
montreal_forced_aligner/transcription/multiprocessing.py
class DecodeArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.DecodeFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    decode_options: MetaDict
    model_path: str
    lat_paths: Dict[str, str]
    word_symbol_paths: Dict[str, str]
    hclg_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class DecodeFunction(KaldiFunction):
    """
    Multiprocessing function for performing decoding

    See Also
    --------
    :meth:`.Transcriber.transcribe`
        Main function that calls this function in parallel
    :meth:`.Transcriber.decode_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`gmm-latgen-faster`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.DecodeArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Log-like per frame for utterance (?P<utterance>.*) is (?P<loglike>[-\d.]+) over (?P<num_frames>\d+) frames."
    )

    def __init__(self, args: DecodeArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.lat_paths = args.lat_paths
        self.word_symbol_paths = args.word_symbol_paths
        self.hclg_paths = args.hclg_paths
        self.decode_options = args.decode_options
        self.model_path = args.model_path

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                feature_string = self.feature_strings[dict_name]
                lat_path = self.lat_paths[dict_name]
                word_symbol_path = self.word_symbol_paths[dict_name]
                hclg_path = self.hclg_paths[dict_name]
                if os.path.exists(lat_path):
                    continue
                if (
                    self.decode_options["uses_speaker_adaptation"]
                    and self.decode_options["first_beam"] is not None
                ):
                    beam = self.decode_options["first_beam"]
                else:
                    beam = self.decode_options["beam"]
                if (
                    self.decode_options["uses_speaker_adaptation"]
                    and self.decode_options["first_max_active"] is not None
                ):
                    max_active = self.decode_options["first_max_active"]
                else:
                    max_active = self.decode_options["max_active"]
                decode_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-latgen-faster"),
                        f"--max-active={max_active}",
                        f"--beam={beam}",
                        f"--lattice-beam={self.decode_options['lattice_beam']}",
                        "--allow-partial=true",
                        f"--word-symbol-table={word_symbol_path}",
                        f"--acoustic-scale={self.decode_options['acoustic_scale']}",
                        self.model_path,
                        hclg_path,
                        feature_string,
                        f"ark:{lat_path}",
                    ],
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in decode_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield m.group("utterance"), m.group("loglike"), m.group("num_frames")
montreal_forced_aligner/helper.py
def parse_old_features(config: MetaDict) -> MetaDict:
    """
    Backwards compatibility function to parse old feature configuration blocks

    Parameters
    ----------
    config: dict[str, Any]
        Configuration parameters

    Returns
    -------
    dict[str, Any]
        Up to date versions of feature blocks
    """
    feature_key_remapping = {
        "type": "feature_type",
        "deltas": "uses_deltas",
    }
    skip_keys = ["lda", "fmllr"]
    if "features" in config:
        for key in skip_keys:
            if key in config["features"]:
                del config["features"][key]
        for key, new_key in feature_key_remapping.items():
            if key in config["features"]:

                config["features"][new_key] = config["features"][key]
                del config["features"][key]
    else:
        for key in skip_keys:
            if key in config:
                del config[key]
        for key, new_key in feature_key_remapping.items():
            if key in config:
                config[new_key] = config[key]
                del config[key]
    return config
montreal_forced_aligner/utils.py
def log_kaldi_errors(error_logs: List[str], logger: logging.Logger) -> None:
    """
    Save details of Kaldi processing errors to a logger

    Parameters
    ----------
    error_logs: list[str]
        Kaldi log files with errors
    logger: :class:`~logging.Logger`
        Logger to output to
    """
    logger.debug(f"There were {len(error_logs)} kaldi processing files that had errors:")
    for path in error_logs:
        logger.debug("")
        logger.debug(path)
        with open(path, "r", encoding="utf8") as f:
            for line in f:
                logger.debug("\t" + line.strip())
montreal_forced_aligner/utils.py
class Stopped(object):
    """
    Multiprocessing class for detecting whether processes should stop processing and exit ASAP

    Attributes
    ----------
    val: :func:`~multiprocessing.Value`
        0 if not stopped, 1 if stopped
    lock: :class:`~multiprocessing.Lock`
        Lock for process safety
    _source: multiprocessing.Value
        1 if it was a Ctrl+C event that stopped it, 0 otherwise
    """

    def __init__(self, initval: Union[bool, int] = False):
        self.val = mp.Value("i", initval)
        self.lock = mp.Lock()
        self._source = mp.Value("i", 0)

    def stop(self) -> None:
        """Signal that work should stop asap"""
        with self.lock:
            self.val.value = True

    def stop_check(self) -> int:
        """Check whether a process should stop"""
        with self.lock:
            return self.val.value

    def set_sigint_source(self) -> None:
        """Set the source as a ctrl+c"""
        with self.lock:
            self._source.value = True

    def source(self) -> int:
        """Get the source value"""
        with self.lock:
            return self._source.value
montreal_forced_aligner/transcription/multiprocessing.py
class CreateHclgFunction(KaldiFunction):
    """
    Create HCLG.fst file

    See Also
    --------
    :meth:`.Transcriber.create_hclgs`
        Main function that calls this function in parallel
    :meth:`.Transcriber.create_hclgs_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`add-self-loops`
        Relevant Kaldi binary
    :openfst_src:`fstconvert`
        Relevant OpenFst binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.CreateHclgArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Log-like per frame for utterance (?P<utterance>.*) is (?P<loglike>[-\d.]+) over (?P<num_frames>\d+) frames."
    )

    def __init__(self, args: CreateHclgArguments):
        self.log_path = args.log_path
        self.working_directory = args.working_directory
        self.path_template = args.path_template
        self.words_path = args.words_path
        self.carpa_path = args.carpa_path
        self.small_arpa_path = args.small_arpa_path
        self.medium_arpa_path = args.medium_arpa_path
        self.big_arpa_path = args.big_arpa_path
        self.model_path = args.model_path
        self.disambig_L_path = args.disambig_L_path
        self.disambig_int_path = args.disambig_int_path
        self.hclg_options = args.hclg_options
        self.words_mapping = args.words_mapping

    def run(self):
        """Run the function"""
        hclg_path = self.path_template.format(file_name="HCLG")
        small_g_path = self.path_template.format(file_name="G.small")
        medium_g_path = self.path_template.format(file_name="G.med")
        lg_path = self.path_template.format(file_name="LG")
        hclga_path = self.path_template.format(file_name="HCLGa")
        if os.path.exists(hclg_path):
            return
        with open(self.log_path, "w") as log_file:
            context_width = self.hclg_options["context_width"]
            central_pos = self.hclg_options["central_pos"]

            clg_path = self.path_template.format(file_name=f"CLG_{context_width}_{central_pos}")
            ilabels_temp = self.path_template.format(
                file_name=f"ilabels_{context_width}_{central_pos}"
            ).replace(".fst", "")
            out_disambig = self.path_template.format(
                file_name=f"disambig_ilabels_{context_width}_{central_pos}"
            ).replace(".fst", ".int")

            log_file.write("Generating decoding graph...\n")
            if not os.path.exists(small_g_path):
                log_file.write("Generating small_G.fst...")
                compose_g(self.small_arpa_path, self.words_path, small_g_path, log_file)
            if not os.path.exists(medium_g_path):
                log_file.write("Generating med_G.fst...")
                compose_g(self.medium_arpa_path, self.words_path, medium_g_path, log_file)
            if not os.path.exists(self.carpa_path):
                log_file.write("Generating G.carpa...")
                temp_carpa_path = self.carpa_path + ".temp"
                compose_g_carpa(
                    self.big_arpa_path,
                    temp_carpa_path,
                    self.words_mapping,
                    self.carpa_path,
                    log_file,
                )
            if not os.path.exists(lg_path):
                log_file.write("Generating LG.fst...")
                compose_lg(self.disambig_L_path, small_g_path, lg_path, log_file)
            if not os.path.exists(clg_path):
                log_file.write("Generating CLG.fst...")
                compose_clg(
                    self.disambig_int_path,
                    out_disambig,
                    context_width,
                    central_pos,
                    ilabels_temp,
                    lg_path,
                    clg_path,
                    log_file,
                )
            if not os.path.exists(hclga_path):
                log_file.write("Generating HCLGa.fst...")
                compose_hclg(
                    self.working_directory,
                    ilabels_temp,
                    self.hclg_options["transition_scale"],
                    clg_path,
                    hclga_path,
                    log_file,
                )
            log_file.write("Generating HCLG.fst...")
            self_loop_proc = subprocess.Popen(
                [
                    thirdparty_binary("add-self-loops"),
                    f"--self-loop-scale={self.hclg_options['self_loop_scale']}",
                    "--reorder=true",
                    self.model_path,
                    hclga_path,
                ],
                stderr=log_file,
                stdout=subprocess.PIPE,
                env=os.environ,
            )
            convert_proc = subprocess.Popen(
                [
                    thirdparty_binary("fstconvert"),
                    "--v=100",
                    "--fst_type=const",
                    "-",
                    hclg_path,
                ],
                stdin=self_loop_proc.stdout,
                stderr=log_file,
                env=os.environ,
            )
            convert_proc.communicate()
            if os.path.exists(hclg_path):
                yield True, hclg_path
            else:
                yield False, hclg_path
montreal_forced_aligner/utils.py
class KaldiProcessWorker(mp.Process):
    """
    Multiprocessing function work

    Parameters
    ----------
    job_name: int
        Integer number of job
    job_q: :class:`~multiprocessing.Queue`
        Job queue to pull arguments from
    function: KaldiFunction
        Multiprocessing function to call on arguments from job_q
    return_dict: dict
        Dictionary for collecting errors
    stopped: :class:`~montreal_forced_aligner.utils.Stopped`
        Stop check
    return_info: dict[int, Any], optional
        Optional dictionary to fill if the function should return information to main thread
    """

    def __init__(
        self,
        job_name: int,
        return_q: mp.Queue,
        function: KaldiFunction,
        error_dict: dict,
        stopped: Stopped,
    ):
        mp.Process.__init__(self)
        self.job_name = job_name
        self.function = function
        self.return_q = return_q
        self.error_dict = error_dict
        self.stopped = stopped
        self.finished = Stopped()

    def run(self) -> None:
        """
        Run through the arguments in the queue apply the function to them
        """
        try:
            for result in self.function.run():
                self.return_q.put(result)
        except Exception:
            self.stopped.stop()
            self.error_dict[self.job_name] = Exception(traceback.format_exception(*sys.exc_info()))
        finally:
            self.finished.stop()
montreal_forced_aligner/transcription/multiprocessing.py
class ScoreFunction(KaldiFunction):
    """
    Multiprocessing function for scoring lattices

    See Also
    --------
    :meth:`~montreal_forced_aligner.transcription.Transcriber.score_transcriptions`
        Main function that calls this function in parallel
    :meth:`.Transcriber.score_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`lattice-scale`
        Relevant Kaldi binary
    :kaldi_src:`lattice-add-penalty`
        Relevant Kaldi binary
    :kaldi_src:`lattice-best-path`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.ScoreArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG .* For utterance (?P<utterance>.*), best cost (?P<graph_cost>[-\d.]+) \+ (?P<acoustic_cost>[-\d.]+) = (?P<total_cost>[-\d.]+) over (?P<num_frames>\d+) frames."
    )

    def __init__(self, args: ScoreArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.score_options = args.score_options
        self.lat_paths = args.lat_paths
        self.rescored_lat_paths = args.rescored_lat_paths
        self.carpa_rescored_lat_paths = args.carpa_rescored_lat_paths
        self.words_paths = args.words_paths
        self.tra_paths = args.tra_paths

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                language_model_weight = self.score_options["language_model_weight"]
                word_insertion_penalty = self.score_options["word_insertion_penalty"]
                carpa_rescored_lat_path = self.carpa_rescored_lat_paths[dict_name]
                rescored_lat_path = self.rescored_lat_paths[dict_name]
                lat_path = self.lat_paths[dict_name]
                words_path = self.words_paths[dict_name]
                tra_path = self.tra_paths[dict_name]
                if os.path.exists(carpa_rescored_lat_path):
                    lat_path = carpa_rescored_lat_path
                elif os.path.exists(rescored_lat_path):
                    lat_path = rescored_lat_path
                scale_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-scale"),
                        f"--inv-acoustic-scale={language_model_weight}",
                        f"ark:{lat_path}",
                        "ark:-",
                    ],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                penalty_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-add-penalty"),
                        f"--word-ins-penalty={word_insertion_penalty}",
                        "ark:-",
                        "ark:-",
                    ],
                    stdin=scale_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                best_path_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-best-path"),
                        f"--word-symbol-table={words_path}",
                        "ark:-",
                        f"ark,t:{tra_path}",
                    ],
                    stdin=penalty_proc.stdout,
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in best_path_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield m.group("utterance"), float(m.group("graph_cost")), float(
                            m.group("acoustic_cost")
                        ), float(m.group("total_cost")), int(m.group("num_frames"))
montreal_forced_aligner/abc.py
class FileExporterMixin(ExporterMixin, metaclass=abc.ABCMeta):
    """
    Abstract mixin class for exporting TextGrid and text files

    Parameters
    ----------
    cleanup_textgrids: bool
        Flag for whether to clean up exported TextGrids
    """

    def __init__(self, cleanup_textgrids: bool = True, **kwargs):
        self.cleanup_textgrids = cleanup_textgrids
        super().__init__(**kwargs)

    @property
    def backup_output_directory(self) -> Optional[str]:
        """Path to store files if overwriting is not allowed"""
        if self.overwrite:
            return None
        return os.path.join(self.working_directory, "backup")

    @abc.abstractmethod
    def export_files(self, output_directory: str) -> None:
        """
        Export files to an output directory

        Parameters
        ----------
        output_directory: str
            Directory to export to
        """
        ...
montreal_forced_aligner/transcription/multiprocessing.py
class FmllrRescoreArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.FmllrRescoreFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    model_path: str
    fmllr_options: MetaDict
    tmp_lat_paths: Dict[str, str]
    final_lat_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class CarpaLmRescoreArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.CarpaLmRescoreFunction`"""

    log_path: str
    dictionaries: List[str]
    lat_paths: Dict[str, str]
    rescored_lat_paths: Dict[str, str]
    old_g_paths: Dict[str, str]
    new_g_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class FinalFmllrArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.FinalFmllrFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    model_path: str
    fmllr_options: MetaDict
    trans_paths: Dict[str, str]
    spk2utt_paths: Dict[str, str]
    tmp_lat_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class FinalFmllrFunction(KaldiFunction):

    """
    Multiprocessing function for running final fMLLR estimation

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe_fmllr`
        Main function that calls this function in parallel
    :meth:`.Transcriber.final_fmllr_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`lattice-determinize-pruned`
        Relevant Kaldi binary
    :kaldi_src:`lattice-to-post`
        Relevant Kaldi binary
    :kaldi_src:`weight-silence-post`
        Relevant Kaldi binary
    :kaldi_src:`gmm-est-fmllr`
        Relevant Kaldi binary
    :kaldi_src:`compose-transforms`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.FinalFmllrArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Done (?P<done>\d+) files, (?P<no_gpost>\d+) with no g?posts, (?P<other_errors>\d+) with other errors."
    )

    def __init__(self, args: FinalFmllrArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.model_path = args.model_path
        self.fmllr_options = args.fmllr_options
        self.trans_paths = args.trans_paths
        self.tmp_lat_paths = args.tmp_lat_paths
        self.spk2utt_paths = args.spk2utt_paths

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                feature_string = self.feature_strings[dict_name]
                trans_path = self.trans_paths[dict_name]
                temp_trans_path = trans_path + ".temp"
                temp_composed_trans_path = trans_path + ".temp_composed"
                spk2utt_path = self.spk2utt_paths[dict_name]
                tmp_lat_path = self.tmp_lat_paths[dict_name]
                determinize_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-determinize-pruned"),
                        f"--acoustic-scale={self.fmllr_options['acoustic_scale']}",
                        "--beam=4.0",
                        f"ark:{tmp_lat_path}",
                        "ark:-",
                    ],
                    stderr=log_file,
                    stdout=subprocess.PIPE,
                    env=os.environ,
                )

                latt_post_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-to-post"),
                        f"--acoustic-scale={self.fmllr_options['acoustic_scale']}",
                        "ark:-",
                        "ark:-",
                    ],
                    stdin=determinize_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                weight_silence_proc = subprocess.Popen(
                    [
                        thirdparty_binary("weight-silence-post"),
                        f"{self.fmllr_options['silence_weight']}",
                        self.fmllr_options["sil_phones"],
                        self.model_path,
                        "ark:-",
                        "ark:-",
                    ],
                    stdin=latt_post_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                fmllr_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-est-fmllr"),
                        f"--fmllr-update-type={self.fmllr_options['fmllr_update_type']}",
                        f"--spk2utt=ark:{spk2utt_path}",
                        self.model_path,
                        feature_string,
                        "ark,s,cs:-",
                        f"ark:{temp_trans_path}",
                    ],
                    stdin=weight_silence_proc.stdout,
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in fmllr_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield int(m.group("done")), int(m.group("no_gpost")), int(
                            m.group("other_errors")
                        )

                compose_proc = subprocess.Popen(
                    [
                        thirdparty_binary("compose-transforms"),
                        "--b-is-affine=true",
                        f"ark:{temp_trans_path}",
                        f"ark:{trans_path}",
                        f"ark:{temp_composed_trans_path}",
                    ],
                    stderr=log_file,
                    stdin=fmllr_proc.stdout,
                    env=os.environ,
                )
                compose_proc.communicate()
                os.remove(trans_path)
                os.remove(temp_trans_path)
                os.rename(temp_composed_trans_path, trans_path)
montreal_forced_aligner/transcription/multiprocessing.py
class InitialFmllrArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.InitialFmllrFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    model_path: str
    fmllr_options: MetaDict
    pre_trans_paths: Dict[str, str]
    lat_paths: Dict[str, str]
    spk2utt_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class CreateHclgArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.CreateHclgFunction`"""

    log_path: str
    working_directory: str
    path_template: str
    words_path: str
    carpa_path: str
    small_arpa_path: str
    medium_arpa_path: str
    big_arpa_path: str
    model_path: str
    disambig_L_path: str
    disambig_int_path: str
    hclg_options: MetaDict
    words_mapping: MappingType

    @property
    def hclg_path(self) -> str:
        """Path to HCLG FST file"""
        return self.path_template.format(file_name="HCLG")
montreal_forced_aligner/abc.py
class TopLevelMfaWorker(MfaWorker, TemporaryDirectoryMixin, metaclass=abc.ABCMeta):
    """
    Abstract mixin for top-level workers in MFA.  This class holds properties about the larger workflow run.

    Parameters
    ----------
    num_jobs: int
        Number of jobs and processes to uses
    clean: bool
        Flag for whether to remove any old files in the work directory
    """

    def __init__(
        self,
        num_jobs: int = 3,
        clean: bool = False,
        **kwargs,
    ):
        kwargs, skipped = type(self).extract_relevant_parameters(kwargs)
        super().__init__(**kwargs)
        self.num_jobs = num_jobs
        self.clean = clean
        self.initialized = False
        self.start_time = time.time()
        self.setup_logger()
        if skipped:
            self.logger.warning(f"Skipped the following configuration keys: {comma_join(skipped)}")

    def __del__(self):
        """Ensure that loggers are cleaned up on delete"""
        handlers = self.logger.handlers[:]
        for handler in handlers:
            handler.close()
            self.logger.removeHandler(handler)

    @abc.abstractmethod
    def setup(self) -> None:
        """Abstract method for setting up a top-level worker"""
        ...

    @property
    def working_directory(self) -> str:
        """Alias for a folder that contains worker information, separate from the data directory"""
        return self.workflow_directory

    @classmethod
    def parse_args(cls, args: Optional[Namespace], unknown_args: Optional[List[str]]) -> MetaDict:
        """
        Class method for parsing configuration parameters from command line arguments

        Parameters
        ----------
        args: :class:`~argparse.Namespace`
            Arguments parsed by argparse
        unknown_args: list[str]
            Optional list of arguments that were not parsed by argparse

        Returns
        -------
        dict[str, Any]
            Dictionary of specified configuration parameters
        """
        param_types = cls.get_configuration_parameters()
        params = {}
        unknown_dict = {}
        if unknown_args:
            for i, a in enumerate(unknown_args):
                if not a.startswith("--"):
                    continue
                name = a.replace("--", "")
                if name not in param_types:
                    continue
                if i == len(unknown_args) - 1 or unknown_args[i + 1].startswith("--"):
                    val = True
                else:
                    val = unknown_args[i + 1]
                unknown_dict[name] = val
        for name, param_type in param_types.items():
            if (name.endswith("_directory") and name != "audio_directory") or name.endswith(
                "_path"
            ):
                continue
            if args is not None and hasattr(args, name):
                params[name] = param_type(getattr(args, name))
            elif name in unknown_dict:
                params[name] = param_type(unknown_dict[name])
                if param_type == bool:
                    if unknown_dict[name].lower() == "false":
                        params[name] = False
        if getattr(args, "disable_mp", False):
            params["use_mp"] = False
        elif getattr(args, "disable_textgrid_cleanup", False):
            params["cleanup_textgrids"] = False
        return params

    @classmethod
    def parse_parameters(
        cls,
        config_path: Optional[str] = None,
        args: Optional[Namespace] = None,
        unknown_args: Optional[List[str]] = None,
    ) -> MetaDict:
        """
        Parse configuration parameters from a config file and command line arguments

        Parameters
        ----------
        config_path: str, optional
            Path to yaml configuration file
        args: :class:`~argparse.Namespace`, optional
            Arguments parsed by argparse
        unknown_args: list[str], optional
            List of unknown arguments from argparse

        Returns
        -------
        dict[str, Any]
            Dictionary of specified configuration parameters
        """
        global_params = {}
        if config_path and os.path.exists(config_path):
            with open(config_path, "r", encoding="utf8") as f:
                data = yaml.load(f, Loader=yaml.SafeLoader)
                for k, v in data.items():
                    if v is None and k in {"punctuation", "compound_markers", "clitic_markers"}:
                        v = []
                    global_params[k] = v
        global_params.update(cls.parse_args(args, unknown_args))
        return global_params

    @property
    @abc.abstractmethod
    def workflow_identifier(self) -> str:
        """Identifier of the worker's workflow"""
        ...

    @property
    def worker_config_path(self):
        """Path to worker's configuration in the working directory"""
        return os.path.join(self.output_directory, f"{self.workflow_identifier}.yaml")

    def cleanup(self) -> None:
        """
        Clean up loggers and output final message for top-level workers
        """
        try:
            if self.dirty:
                self.logger.error("There was an error in the run, please see the log.")
            else:
                self.logger.info(f"Done! Everything took {time.time() - self.start_time} seconds")
            handlers = self.logger.handlers[:]
            for handler in handlers:
                handler.close()
                self.logger.removeHandler(handler)
            self.save_worker_config()
        except (NameError, ValueError):  # already cleaned up
            pass

    def save_worker_config(self):
        """Export worker configuration to its working directory"""
        with open(self.worker_config_path, "w") as f:
            yaml.dump(self.configuration, f)

    def _validate_previous_configuration(self, conf: MetaDict) -> bool:
        """
        Validate the current configuration against a previous configuration

        Parameters
        ----------
        conf: dict[str, Any]
            Previous run's configuration

        Returns
        -------
        bool
            Flag for whether the current run is compatible with the previous one
        """
        from montreal_forced_aligner.utils import get_mfa_version

        clean = True
        current_version = get_mfa_version()
        if conf["dirty"]:
            self.logger.debug("Previous run ended in an error (maybe ctrl-c?)")
            clean = False
        if "type" in conf:
            command = conf["type"]
        elif "command" in conf:
            command = conf["command"]
        else:
            command = self.workflow_identifier
        if command != self.workflow_identifier:
            self.logger.debug(
                f"Previous run was a different subcommand than {self.workflow_identifier} (was {command})"
            )
            clean = False
        if conf.get("version", current_version) != current_version:
            self.logger.debug(
                f"Previous run was on {conf['version']} version (new run: {current_version})"
            )
            clean = False
        for key in [
            "corpus_directory",
            "dictionary_path",
            "acoustic_model_path",
            "g2p_model_path",
            "language_model_path",
        ]:
            if conf.get(key, None) != getattr(self, key, None):
                self.logger.debug(
                    f"Previous run used a different {key.replace('_', ' ')} than {getattr(self, key, None)} (was {conf.get(key, None)})"
                )
                clean = False
        return clean

    def check_previous_run(self) -> bool:
        """
        Check whether a previous run has any conflicting settings with the current run.

        Returns
        -------
        bool
            Flag for whether the current run is compatible with the previous one
        """
        if not os.path.exists(self.worker_config_path):
            return True
        with open(self.worker_config_path, "r") as f:
            conf = yaml.load(f, Loader=yaml.SafeLoader)
        clean = self._validate_previous_configuration(conf)
        if not clean:
            self.logger.warning(
                "The previous run had a different configuration than the current, which may cause issues."
                " Please see the log for details or use --clean flag if issues are encountered."
            )
        return clean

    @property
    def identifier(self) -> str:
        """Combined identifier of the data source and workflow"""
        return f"{self.data_source_identifier}_{self.workflow_identifier}"

    @property
    def output_directory(self) -> str:
        """Root temporary directory to store all of this worker's files"""
        return os.path.join(self.temporary_directory, self.identifier)

    @property
    def workflow_directory(self) -> str:
        """Temporary directory to save work specific to the worker (i.e., not data)"""
        return os.path.join(self.output_directory, self.workflow_identifier)

    @property
    def log_file(self):
        """Path to the worker's log file"""
        return os.path.join(self.output_directory, f"{self.workflow_identifier}.log")

    def setup_logger(self):
        """
        Construct a logger for a command line run
        """
        from .utils import CustomFormatter, get_mfa_version

        current_version = get_mfa_version()
        # Remove previous directory if versions are different
        if os.path.exists(self.worker_config_path):
            with open(self.worker_config_path, "r") as f:
                conf = yaml.load(f, Loader=yaml.SafeLoader)
            if conf.get("version", current_version) != current_version:
                self.clean = True
        if self.clean:
            shutil.rmtree(self.output_directory, ignore_errors=True)
        os.makedirs(self.workflow_directory, exist_ok=True)
        if os.path.exists(self.log_file):
            os.remove(self.log_file)
        self.logger = logging.getLogger(self.workflow_identifier)
        self.logger.setLevel(logging.DEBUG)

        file_handler = logging.FileHandler(self.log_file, encoding="utf8")
        file_handler.setLevel(logging.DEBUG)
        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
        file_handler.setFormatter(formatter)
        self.logger.addHandler(file_handler)

        handler = logging.StreamHandler(sys.stdout)
        if self.verbose:
            handler.setLevel(logging.DEBUG)
        else:
            handler.setLevel(logging.INFO)
        handler.setFormatter(CustomFormatter())
        self.logger.addHandler(handler)
        self.logger.debug(
            f"Beginning run for {self.workflow_identifier} on {self.data_source_identifier}"
        )
        if self.use_mp:
            self.logger.debug(f"Using multiprocessing with {self.num_jobs}")
        else:
            self.logger.debug(f"NOT using multiprocessing with {self.num_jobs}")
        self.logger.debug(f"Set up logger for MFA version: {current_version}")
        if self.clean:
            self.logger.debug("Cleaned previous run")

    def log_debug(self, message: str) -> None:
        """
        Log a debug message. This function is a wrapper around the :meth:`logging.Logger.debug`

        Parameters
        ----------
        message: str
            Debug message to log
        """
        self.logger.debug(message)

    def log_info(self, message: str) -> None:
        """
        Log an info message. This function is a wrapper around the :meth:`logging.Logger.info`

        Parameters
        ----------
        message: str
            Info message to log
        """
        self.logger.info(message)

    def log_warning(self, message: str) -> None:
        """
        Log a warning message. This function is a wrapper around the :meth:`logging.Logger.warning`

        Parameters
        ----------
        message: str
            Warning message to log
        """
        self.logger.warning(message)

    def log_error(self, message: str) -> None:
        """
        Log an error message. This function is a wrapper around the :meth:`logging.Logger.error`

        Parameters
        ----------
        message: str
            Error message to log
        """
        self.logger.error(message)
montreal_forced_aligner/helper.py
def score_wer(gold: List[str], hypo: List[str]) -> Tuple[int, int, int, int]:
    """
    Computes word error rate and character error rate for a transcription

    Parameters
    ----------
    gold: list[str]
        The reference words
    hypo: list[str]
        The hypothesized words

    Returns
    -------
    int
        Word Edit distance
    int
        Length of the gold words labels
    int
        Character edit distance
    int
        Length of the gold characters
    """
    word_edits = edit_distance(gold, hypo)
    character_gold = list("".join(gold))
    character_hypo = list("".join(hypo))
    character_edits = edit_distance(character_gold, character_hypo)
    return word_edits, len(gold), character_edits, len(character_gold)
montreal_forced_aligner/transcription/multiprocessing.py
class LatGenFmllrArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.LatGenFmllrFunction`"""

    log_path: str
    dictionaries: List[str]
    feature_strings: Dict[str, str]
    model_path: str
    decode_options: MetaDict
    word_symbol_paths: Dict[str, str]
    hclg_paths: Dict[str, str]
    tmp_lat_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class InitialFmllrFunction(KaldiFunction):
    """
    Multiprocessing function for running initial fMLLR calculation

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe_fmllr`
        Main function that calls this function in parallel
    :meth:`.Transcriber.initial_fmllr_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`lattice-to-post`
        Relevant Kaldi binary
    :kaldi_src:`weight-silence-post`
        Relevant Kaldi binary
    :kaldi_src:`gmm-post-to-gpost`
        Relevant Kaldi binary
    :kaldi_src:`gmm-est-fmllr-gpost`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.InitialFmllrArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Done (?P<done>\d+) files, (?P<no_gpost>\d+) with no g?posts, (?P<other_errors>\d+) with other errors."
    )

    def __init__(self, args: InitialFmllrArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.model_path = args.model_path
        self.fmllr_options = args.fmllr_options
        self.pre_trans_paths = args.pre_trans_paths
        self.lat_paths = args.lat_paths
        self.spk2utt_paths = args.spk2utt_paths

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                lat_path = self.lat_paths[dict_name]
                feature_string = self.feature_strings[dict_name]
                spk2utt_path = self.spk2utt_paths[dict_name]
                trans_path = self.pre_trans_paths[dict_name]

                latt_post_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-to-post"),
                        f"--acoustic-scale={self.fmllr_options['acoustic_scale']}",
                        f"ark:{lat_path}",
                        "ark:-",
                    ],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                weight_silence_proc = subprocess.Popen(
                    [
                        thirdparty_binary("weight-silence-post"),
                        f"{self.fmllr_options['silence_weight']}",
                        self.fmllr_options["sil_phones"],
                        self.model_path,
                        "ark:-",
                        "ark:-",
                    ],
                    stdin=latt_post_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                gmm_gpost_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-post-to-gpost"),
                        self.model_path,
                        feature_string,
                        "ark:-",
                        "ark:-",
                    ],
                    stdin=weight_silence_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                fmllr_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-est-fmllr-gpost"),
                        f"--fmllr-update-type={self.fmllr_options['fmllr_update_type']}",
                        f"--spk2utt=ark:{spk2utt_path}",
                        self.model_path,
                        feature_string,
                        "ark,s,cs:-",
                        f"ark:{trans_path}",
                    ],
                    stdin=gmm_gpost_proc.stdout,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in fmllr_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield int(m.group("done")), int(m.group("no_gpost")), int(
                            m.group("other_errors")
                        )
montreal_forced_aligner/corpus/acoustic_corpus.py
class AcousticCorpusPronunciationMixin(
    AcousticCorpusMixin, MultispeakerDictionaryMixin, metaclass=ABCMeta
):
    """
    Mixin for acoustic corpora with Pronunciation dictionaries

    See Also
    --------
    :class:`~montreal_forced_aligner.corpus.acoustic_corpus.AcousticCorpusMixin`
        For corpus parsing parameters
    :class:`~montreal_forced_aligner.dictionary.multispeaker.MultispeakerDictionaryMixin`
        For dictionary parsing parameters
    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def load_corpus(self) -> None:
        """
        Load the corpus
        """
        all_begin = time.time()
        self.dictionary_setup()
        self.log_debug(f"Using {self.phone_set_type}")
        self.log_debug(f"Loaded dictionary in {time.time() - all_begin}")

        begin = time.time()
        self._load_corpus()
        self.log_debug(f"Loaded corpus in {time.time() - begin}")

        begin = time.time()
        self.set_lexicon_word_set(self.corpus_word_set)
        self.log_debug(f"Set up lexicon word set in {time.time() - begin}")

        begin = time.time()

        self.log_debug("Topology:")
        for k, v in self.kaldi_phones_for_topo.items():
            self.log_debug(f"{k}: {', '.join(v)}")
        self.log_debug("Extra questions:")
        for k, v in self.extra_questions_mapping.items():
            self.log_debug(f"{k}: {', '.join(v)}")
        self.write_lexicon_information()
        self.log_debug(f"Wrote lexicon information in {time.time() - begin}")

        begin = time.time()
        for speaker in self.speakers:
            speaker.set_dictionary(self.get_dictionary(speaker.name))
        self.log_debug(f"Set dictionaries for speakers in {time.time() - begin}")

        begin = time.time()
        self.initialize_jobs()
        self.log_debug(f"Initialized jobs in {time.time() - begin}")
        begin = time.time()
        self.write_corpus_information()
        self.log_debug(f"Wrote corpus information in {time.time() - begin}")

        begin = time.time()
        self.create_corpus_split()
        self.log_debug(f"Created corpus split directory in {time.time() - begin}")

        begin = time.time()
        self.generate_features()
        self.log_debug(f"Generated features in {time.time() - begin}")

        begin = time.time()
        self.calculate_oovs_found()
        self.log_debug(f"Calculated oovs found in {time.time() - begin}")
        self.log_debug(f"Setting up corpus took {time.time() - all_begin} seconds")
montreal_forced_aligner/models.py
class LanguageModel(Archive):
    """
    Class for MFA language models
    """

    model_type = "language_model"

    arpa_extension = ".arpa"
    extensions = [f".{FORMAT}", arpa_extension, ".lm"]

    def __init__(self, source: str, root_directory: Optional[str] = None):
        if source in LanguageModel.get_available_models():
            source = LanguageModel.get_pretrained_path(source)
        from .config import get_temporary_directory

        if root_directory is None:
            root_directory = get_temporary_directory()

        if source.endswith(self.arpa_extension):
            self.root_directory = root_directory
            self._meta = {}
            self.name, _ = os.path.splitext(os.path.basename(source))
            self.dirname = os.path.join(root_directory, self.name)
            if not os.path.exists(self.dirname):
                os.makedirs(self.dirname, exist_ok=True)
            copy(source, self.large_arpa_path)
        else:
            super().__init__(source, root_directory)

    @property
    def decode_arpa_path(self) -> str:
        """
        Uses the smallest language model for decoding
        """
        for path in [self.small_arpa_path, self.medium_arpa_path, self.large_arpa_path]:
            if os.path.exists(path):
                return path
        raise LanguageModelNotFoundError()

    @property
    def carpa_path(self) -> str:
        """
        Uses the largest language model for rescoring
        """
        for path in [self.large_arpa_path, self.medium_arpa_path, self.small_arpa_path]:
            if os.path.exists(path):
                return path
        raise LanguageModelNotFoundError()

    @property
    def small_arpa_path(self) -> str:
        """Small arpa path"""
        return os.path.join(self.dirname, f"{self.name}_small{self.arpa_extension}")

    @property
    def medium_arpa_path(self) -> str:
        """Medium arpa path"""
        return os.path.join(self.dirname, f"{self.name}_med{self.arpa_extension}")

    @property
    def large_arpa_path(self) -> str:
        """Large arpa path"""
        return os.path.join(self.dirname, self.name + self.arpa_extension)

    def add_arpa_file(self, arpa_path: str) -> None:
        """
        Adds an ARPA file to the model

        Parameters
        ----------
        arpa_path: str
            Path to ARPA file
        """
        output_name = self.large_arpa_path
        if arpa_path.endswith("_small.arpa"):
            output_name = self.small_arpa_path
        elif arpa_path.endswith("_medium.arpa"):
            output_name = self.medium_arpa_path
        copyfile(arpa_path, output_name)
montreal_forced_aligner/transcription/multiprocessing.py
class LmRescoreFunction(KaldiFunction):
    """
    Multiprocessing function rescore lattices by replacing the small G.fst with the medium G.fst

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe`
        Main function that calls this function in parallel
    :meth:`.Transcriber.lm_rescore_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`lattice-lmrescore-pruned`
        Relevant Kaldi binary
    :openfst_src:`fstproject`
        Relevant OpenFst binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.LmRescoreArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG .* Overall, succeeded for (?P<succeeded>\d+) lattices, failed for (?P<failed>\d+)"
    )

    def __init__(self, args: LmRescoreArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.lat_paths = args.lat_paths
        self.rescored_lat_paths = args.rescored_lat_paths
        self.old_g_paths = args.old_g_paths
        self.new_g_paths = args.new_g_paths
        self.lm_rescore_options = args.lm_rescore_options

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                lat_path = self.lat_paths[dict_name]
                rescored_lat_path = self.rescored_lat_paths[dict_name]
                old_g_path = self.old_g_paths[dict_name]
                new_g_path = self.new_g_paths[dict_name]
                if sys.platform == "win32":
                    project_type_arg = "--project_output=true"
                else:
                    project_type_arg = "--project_type=output"
                if os.path.exists(rescored_lat_path):
                    continue

                project_proc = subprocess.Popen(
                    [thirdparty_binary("fstproject"), project_type_arg, old_g_path],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                lattice_scale_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-lmrescore-pruned"),
                        f"--acoustic-scale={self.lm_rescore_options['acoustic_scale']}",
                        "-",
                        f"fstproject {project_type_arg} {new_g_path} |",
                        f"ark:{lat_path}",
                        f"ark:{rescored_lat_path}",
                    ],
                    stdin=project_proc.stdout,
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in lattice_scale_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield int(m.group("succeeded")), int(m.group("failed"))
montreal_forced_aligner/transcription/multiprocessing.py
class LmRescoreArguments(NamedTuple):
    """Arguments for :class:`~montreal_forced_aligner.transcription.multiprocessing.LmRescoreFunction`"""

    log_path: str
    dictionaries: List[str]
    lm_rescore_options: MetaDict
    lat_paths: Dict[str, str]
    rescored_lat_paths: Dict[str, str]
    old_g_paths: Dict[str, str]
    new_g_paths: Dict[str, str]
montreal_forced_aligner/transcription/multiprocessing.py
class CarpaLmRescoreFunction(KaldiFunction):
    """
    Multiprocessing function to rescore lattices by replacing medium G.fst with large G.carpa

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe`
        Main function that calls this function in parallel
    :meth:`.Transcriber.carpa_lm_rescore_arguments`
        Job method for generating arguments for this function
    :openfst_src:`fstproject`
        Relevant OpenFst binary
    :kaldi_src:`lattice-lmrescore`
        Relevant Kaldi binary
    :kaldi_src:`lattice-lmrescore-const-arpa`
        Relevant Kaldi binary

    Parameters
    ----------
    args: CarpaLmRescoreArguments
        Arguments
    """

    progress_pattern = re.compile(
        r"^LOG .* Overall, succeeded for (?P<succeeded>\d+) lattices, failed for (?P<failed>\d+)"
    )

    def __init__(self, args: CarpaLmRescoreArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.lat_paths = args.lat_paths
        self.rescored_lat_paths = args.rescored_lat_paths
        self.old_g_paths = args.old_g_paths
        self.new_g_paths = args.new_g_paths

    def run(self):
        """Run the function"""
        with open(self.log_path, "a", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                if sys.platform == "win32":
                    project_type_arg = "--project_output=true"
                else:
                    project_type_arg = "--project_type=output"
                lat_path = self.lat_paths[dict_name]
                rescored_lat_path = self.rescored_lat_paths[dict_name]
                old_g_path = self.old_g_paths[dict_name]
                new_g_path = self.new_g_paths[dict_name]
                if os.path.exists(rescored_lat_path):
                    continue
                project_proc = subprocess.Popen(
                    [thirdparty_binary("fstproject"), project_type_arg, old_g_path],
                    stdout=subprocess.PIPE,
                    stderr=log_file,
                    env=os.environ,
                )
                lmrescore_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-lmrescore"),
                        "--lm-scale=-1.0",
                        f"ark:{lat_path}",
                        "-",
                        "ark:-",
                    ],
                    stdout=subprocess.PIPE,
                    stdin=project_proc.stdout,
                    stderr=log_file,
                    env=os.environ,
                )
                lmrescore_const_proc = subprocess.Popen(
                    [
                        thirdparty_binary("lattice-lmrescore-const-arpa"),
                        "--lm-scale=1.0",
                        "ark:-",
                        new_g_path,
                        f"ark:{rescored_lat_path}",
                    ],
                    stdin=lmrescore_proc.stdout,
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in lmrescore_const_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield int(m.group("succeeded")), int(m.group("failed"))
montreal_forced_aligner/utils.py
def thirdparty_binary(binary_name: str) -> str:
    """
    Generate full path to a given binary name

    Notes
    -----
    With the move to conda, this function is deprecated as conda will manage the path much better

    Parameters
    ----------
    binary_name: str
        Executable to run

    Returns
    -------
    str
        Full path to the executable
    """
    bin_path = shutil.which(binary_name)
    if bin_path is None:
        if binary_name in ["fstcompile", "fstarcsort", "fstconvert"] and sys.platform != "win32":
            raise ThirdpartyError(binary_name, open_fst=True)
        else:
            raise ThirdpartyError(binary_name)
    return bin_path
montreal_forced_aligner/models.py
class AcousticModel(Archive):
    """
    Class for storing acoustic models in MFA, exported as zip files containing the necessary Kaldi files
    to be reused

    """

    files = ["final.mdl", "final.alimdl", "final.occs", "lda.mat", "tree"]
    extensions = [".zip", ".am"]

    model_type = "acoustic"

    def __init__(self, source: str, root_directory: Optional[str] = None):
        if source in AcousticModel.get_available_models():
            source = AcousticModel.get_pretrained_path(source)

        super().__init__(source, root_directory)

    def add_meta_file(self, trainer: ModelExporterMixin) -> None:
        """
        Add metadata file from a model trainer

        Parameters
        ----------
        trainer: :class:`~montreal_forced_aligner.abc.ModelExporterMixin`
            Trainer to supply metadata information about the acoustic model
        """
        with open(os.path.join(self.dirname, "meta.json"), "w", encoding="utf8") as f:
            json.dump(trainer.meta, f)

    @property
    def parameters(self) -> MetaDict:
        """Parameters to pass to top-level workers"""
        params = {**self.meta["features"]}
        params["non_silence_phones"] = {x for x in self.meta["phones"]}
        params["oov_phone"] = self.meta["oov_phone"]
        params["optional_silence_phone"] = self.meta["optional_silence_phone"]
        params["other_noise_phone"] = self.meta["other_noise_phone"]
        params["phone_set_type"] = self.meta["phone_set_type"]
        return params

    @property
    def meta(self) -> MetaDict:
        """
        Metadata information for the acoustic model
        """
        default_features = {
            "feature_type": "mfcc",
            "use_energy": False,
            "frame_shift": 10,
            "snip_edges": True,
            "low_frequency": 20,
            "high_frequency": 7800,
            "sample_frequency": 16000,
            "allow_downsample": True,
            "allow_upsample": True,
            "pitch": False,
            "uses_cmvn": True,
            "uses_deltas": True,
            "uses_splices": False,
            "uses_voiced": False,
            "uses_speaker_adaptation": False,
            "silence_weight": 0.0,
            "fmllr_update_type": "full",
            "splice_left_context": 3,
            "splice_right_context": 3,
        }
        if not self._meta:
            meta_path = os.path.join(self.dirname, "meta.json")
            format = "json"
            if not os.path.exists(meta_path):
                meta_path = os.path.join(self.dirname, "meta.yaml")
                format = "yaml"
            if not os.path.exists(meta_path):
                self._meta = {
                    "version": "0.9.0",
                    "architecture": "gmm-hmm",
                    "features": default_features,
                }
            else:
                with open(meta_path, "r", encoding="utf8") as f:
                    if format == "yaml":
                        self._meta = yaml.safe_load(f)
                    else:
                        self._meta = json.load(f)
                if self._meta["features"] == "mfcc+deltas":
                    self._meta["features"] = default_features
            if "phone_type" not in self._meta:
                self._meta["phone_type"] = "triphone"
            if "optional_silence_phone" not in self._meta:
                self._meta["optional_silence_phone"] = "sil"
            if "oov_phone" not in self._meta:
                self._meta["oov_phone"] = "spn"
            if "other_noise_phone" not in self._meta:
                self._meta["other_noise_phone"] = "sp"
            if "phone_set_type" not in self._meta:
                self._meta["phone_set_type"] = "UNKNOWN"
            self._meta["phones"] = set(self._meta.get("phones", []))
            if (
                "uses_speaker_adaptation" not in self._meta["features"]
                or not self._meta["features"]["uses_speaker_adaptation"]
            ):
                self._meta["features"]["uses_speaker_adaptation"] = os.path.exists(
                    os.path.join(self.dirname, "final.alimdl")
                )
            if (
                "uses_splices" not in self._meta["features"]
                or not self._meta["features"]["uses_splices"]
            ):
                self._meta["features"]["uses_splices"] = os.path.exists(
                    os.path.join(self.dirname, "lda.mat")
                )
                if self._meta["features"]["uses_splices"]:
                    self._meta["features"]["uses_deltas"] = False
        self.parse_old_features()
        return self._meta

    def pretty_print(self) -> None:
        """
        Prints the metadata information to the terminal
        """
        from .utils import get_mfa_version

        printer = TerminalPrinter()
        configuration_data = {"Acoustic model": {"name": (self.name, "green"), "data": {}}}
        version_color = "green"
        if self.meta["version"] != get_mfa_version():
            version_color = "red"
        configuration_data["Acoustic model"]["data"]["Version"] = (
            self.meta["version"],
            version_color,
        )

        if "citation" in self.meta:
            configuration_data["Acoustic model"]["data"]["Citation"] = self.meta["citation"]
        if "train_date" in self.meta:
            configuration_data["Acoustic model"]["data"]["Train date"] = self.meta["train_date"]
        configuration_data["Acoustic model"]["data"]["Architecture"] = self.meta["architecture"]
        configuration_data["Acoustic model"]["data"]["Phone type"] = self.meta["phone_type"]
        configuration_data["Acoustic model"]["data"]["Features"] = {
            "Feature type": self.meta["features"]["feature_type"],
            "Frame shift": self.meta["features"]["frame_shift"],
            "Performs speaker adaptation": self.meta["features"]["uses_speaker_adaptation"],
            "Performs LDA on features": self.meta["features"]["uses_splices"],
        }
        if self.meta["phones"]:
            configuration_data["Acoustic model"]["data"]["Phones"] = self.meta["phones"]
        else:
            configuration_data["Acoustic model"]["data"]["Phones"] = ("None found!", "red")

        printer.print_config(configuration_data)

    def add_model(self, source: str) -> None:
        """
        Add file into archive

        Parameters
        ----------
        source: str
            File to add
        """
        for f in self.files:
            if os.path.exists(os.path.join(source, f)):
                copyfile(os.path.join(source, f), os.path.join(self.dirname, f))

    def export_model(self, destination: str) -> None:
        """
        Extract the model files to a new directory

        Parameters
        ----------
        destination: str
            Destination directory to extract files to
        """
        os.makedirs(destination, exist_ok=True)
        for f in self.files:
            if os.path.exists(os.path.join(self.dirname, f)):
                copyfile(os.path.join(self.dirname, f), os.path.join(destination, f))

    def log_details(self, logger: Logger) -> None:
        """
        Log metadata information to a logger

        Parameters
        ----------
        logger: :class:`~logging.Logger`
            Logger to send debug information to
        """
        logger.debug("")
        logger.debug("====ACOUSTIC MODEL INFO====")
        logger.debug("Acoustic model root directory: " + self.root_directory)
        logger.debug("Acoustic model dirname: " + self.dirname)
        meta_path = os.path.join(self.dirname, "meta.json")
        if not os.path.exists(meta_path):
            meta_path = os.path.join(self.dirname, "meta.yaml")
        logger.debug("Acoustic model meta path: " + meta_path)
        if not os.path.exists(meta_path):
            logger.debug("META.YAML DOES NOT EXIST, this may cause issues in validating the model")
        logger.debug("Acoustic model meta information:")
        stream = yaml.dump(self.meta)
        logger.debug(stream)
        logger.debug("")

    def validate(self, dictionary: DictionaryMixin) -> None:
        """
        Validate this acoustic model against a pronunciation dictionary to ensure their
        phone sets are compatible

        Parameters
        ----------
        dictionary: :class:`~montreal_forced_aligner.dictionary.mixins.DictionaryMixin`
            DictionaryMixin  to compare phone sets with

        Raises
        ------
        :class:`~montreal_forced_aligner.exceptions.PronunciationAcousticMismatchError`
            If there are phones missing from the acoustic model
        """
        if isinstance(dictionary, G2PModel):
            missing_phones = dictionary.meta["phones"] - set(self.meta["phones"])
        else:
            missing_phones = dictionary.non_silence_phones - set(self.meta["phones"])
        if missing_phones:
            raise (PronunciationAcousticMismatchError(missing_phones))
montreal_forced_aligner/transcription/multiprocessing.py
class LatGenFmllrFunction(KaldiFunction):
    """
    Regenerate lattices using initial fMLLR transforms

    See Also
    --------
    :func:`~montreal_forced_aligner.transcription.Transcriber.transcribe_fmllr`
        Main function that calls this function in parallel
    :meth:`.Transcriber.lat_gen_fmllr_arguments`
        Job method for generating arguments for this function
    :kaldi_src:`gmm-latgen-faster`
        Relevant Kaldi binary

    Parameters
    ----------
    args: :class:`~montreal_forced_aligner.transcription.multiprocessing.LatGenFmllrArguments`
        Arguments for the function
    """

    progress_pattern = re.compile(
        r"^LOG.*Log-like per frame for utterance (?P<utterance>.*) is (?P<loglike>[-\d.]+) over (?P<num_frames>\d+) frames."
    )

    def __init__(self, args: LatGenFmllrArguments):
        self.log_path = args.log_path
        self.dictionaries = args.dictionaries
        self.feature_strings = args.feature_strings
        self.tmp_lat_paths = args.tmp_lat_paths
        self.word_symbol_paths = args.word_symbol_paths
        self.hclg_paths = args.hclg_paths
        self.decode_options = args.decode_options
        self.model_path = args.model_path

    def run(self):
        """Run the function"""
        with open(self.log_path, "w", encoding="utf8") as log_file:
            for dict_name in self.dictionaries:
                feature_string = self.feature_strings[dict_name]
                words_path = self.word_symbol_paths[dict_name]
                hclg_path = self.hclg_paths[dict_name]
                tmp_lat_path = self.tmp_lat_paths[dict_name]
                lat_gen_proc = subprocess.Popen(
                    [
                        thirdparty_binary("gmm-latgen-faster"),
                        f"--max-active={self.decode_options['max_active']}",
                        f"--beam={self.decode_options['beam']}",
                        f"--lattice-beam={self.decode_options['lattice_beam']}",
                        f"--acoustic-scale={self.decode_options['acoustic_scale']}",
                        "--determinize-lattice=false",
                        "--allow-partial=true",
                        f"--word-symbol-table={words_path}",
                        self.model_path,
                        hclg_path,
                        feature_string,
                        f"ark:{tmp_lat_path}",
                    ],
                    stderr=subprocess.PIPE,
                    env=os.environ,
                    encoding="utf8",
                )
                for line in lat_gen_proc.stderr:
                    log_file.write(line)
                    m = self.progress_pattern.match(line.strip())
                    if m:
                        yield m.group("utterance"), m.group("loglike"), m.group("num_frames")
montreal_forced_aligner/exceptions.py
class KaldiProcessingError(MFAError):
    """
    Exception class for when a Kaldi binary has an exception

    Parameters
    ----------
    error_logs: list[str]
        List of Kaldi logs that had errors
    log_file: str, optional
        Overall log file to find more information
    """

    def __init__(self, error_logs: List[str], log_file: Optional[str] = None):
        super().__init__(
            f"There were {len(error_logs)} job(s) with errors when running Kaldi binaries."
        )

        if log_file is not None:
            self.message_lines.append(
                f" For more details, please check {self.printer.error_text(log_file)}"
            )
        self.error_logs = error_logs
        self.log_file = log_file

    def update_log_file(self, logger: logging.Logger) -> None:
        """
        Update the log file output

        Parameters
        ----------
        logger: logging.Logger
            Logger
        """
        if logger.handlers:
            self.log_file = logger.handlers[0].baseFilename
        self.message_lines = [
            f"There were {len(self.error_logs)} job(s) with errors when running Kaldi binaries."
        ]
        if self.log_file is not None:
            self.message_lines.append(
                f" For more details, please check {self.printer.error_text(self.log_file)}"
            )


Question: import csv
import itertools
import multiprocessing as mp
import os
import re
import shutil
import subprocess
import sys
import time
import tqdm
import yaml
from abc import abstractmethod
from queue import Empty
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from montreal_forced_aligner.abc import FileExporterMixin, TopLevelMfaWorker
from montreal_forced_aligner.corpus.acoustic_corpus import AcousticCorpusPronunciationMixin
from montreal_forced_aligner.exceptions import KaldiProcessingError, PlatformError
from montreal_forced_aligner.helper import parse_old_features, score_wer
from montreal_forced_aligner.models import AcousticModel, LanguageModel
from montreal_forced_aligner.transcription.multiprocessing import (
    CarpaLmRescoreArguments,
    CarpaLmRescoreFunction,
    CreateHclgArguments,
    CreateHclgFunction,
    DecodeArguments,
    DecodeFunction,
    FinalFmllrArguments,
    FinalFmllrFunction,
    FmllrRescoreArguments,
    FmllrRescoreFunction,
    InitialFmllrArguments,
    InitialFmllrFunction,
    LatGenFmllrArguments,
    LatGenFmllrFunction,
    LmRescoreArguments,
    LmRescoreFunction,
    ScoreArguments,
    ScoreFunction,
)
from montreal_forced_aligner.utils import (
    KaldiProcessWorker,
    Stopped,
    log_kaldi_errors,
    thirdparty_binary,
)
    from argparse import Namespace
    from montreal_forced_aligner.abc import MetaDict
            Arguments for function
        """
        self.logger.info("Regenerating lattices with fMLLR transforms...")
        with tqdm.tqdm(total=self.num_utterances) as pbar, open(
            os.path.join(self.working_log_directory, "lat_gen_fmllr_log_like.csv"),
            "w",
            encoding="utf8",
        ) as log_file:
            log_file.write("utterance,log_likelihood,num_frames\n")
            if self.use_mp:
                manager = mp.Manager()
                error_dict = manager.dict()
                return_queue = manager.Queue()
                stopped = Stopped()
                procs = []
                for i, args in enumerate(self.lat_gen_fmllr_arguments()):
                    function = LatGenFmllrFunction(args)
                    p = KaldiProcessWorker(i, return_queue, function, error_dict, stopped)
                    procs.append(p)
                    p.start()
                while True:
                    try:
                        utterance, log_likelihood, num_frames = return_queue.get(timeout=1)
                        log_file.write(f"{utterance},{log_likelihood},{num_frames}\n")
                        if stopped.stop_check():
                            continue
                    except Empty:
                        for proc in procs:
                            if not proc.finished.stop_check():
                                break
                        else:
                            break
                        continue
                    pbar.update(1)
                for p in procs:
                    p.join()
                if error_dict:
                    for v in error_dict.values():
                        raise v
            else:
                for args in self.lat_gen_fmllr_arguments():
                    function = LatGenFmllrFunction(args)
                    for utterance, log_likelihood, num_frames in function.run():
                        log_file.write(f"{utterance},{log_likelihood},{num_frames}\n")
                        pbar.update(1)

    def calc_final_fmllr(self):
        """
        Calculate final fMLLR transforms

        See Also
        -------
        :class:`~montreal_forced_aligner.transcription.multiprocessing.FinalFmllrFunction`
            Multiprocessing function
        :meth:`.Transcriber.final_fmllr_arguments`
            Arguments for function
        """
        self.logger.info("Calculating final fMLLR transforms...")
        sum_errors = 0
        with tqdm.tqdm(total=self.num_utterances) as pbar:
            if self.use_mp:
                manager = mp.Manager()
                error_dict = manager.dict()
                return_queue = manager.Queue()
                stopped = Stopped()
                procs = []
                for i, args in enumerate(self.final_fmllr_arguments()):
                    function = FinalFmllrFunction(args)
                    p = KaldiProcessWorker(i, return_queue, function, error_dict, stopped)
                    procs.append(p)
                    p.start()
                while True:
                    try:
                        done, no_gpost, other_errors = return_queue.get(timeout=1)
                        sum_errors += no_gpost + other_errors
                        if stopped.stop_check():
                            continue
                    except Empty:
                        for proc in procs:
                            if not proc.finished.stop_check():
                                break
                        else:
                            break
                        continue
                    pbar.update(done + no_gpost + other_errors)
                for p in procs:
                    p.join()
                if error_dict:
                    for v in error_dict.values():
                        raise v
            else:
                for args in self.final_fmllr_arguments():
                    function = FinalFmllrFunction(args)
                    for done, no_gpost, other_errors in function.run():
                        sum_errors += no_gpost + other_errors
                        pbar.update(done + no_gpost + other_errors)
            if sum_errors:
                self.logger.warning(f"{sum_errors} utterances had errors on calculating fMLLR.")

    def fmllr_rescore(self):
        """
        Rescore lattices with final fMLLR transforms

        See Also
        -------
        :class:`~montreal_forced_aligner.transcription.multiprocessing.FmllrRescoreFunction`
            Multiprocessing function
        :meth:`.Transcriber.fmllr_rescore_arguments`
            Arguments for function
        """
        self.logger.info("Rescoring fMLLR lattices with final transform...")
        sum_errors = 0
        with tqdm.tqdm(total=self.num_utterances) as pbar:
            if self.use_mp:
                manager = mp.Manager()
                error_dict = manager.dict()
                return_queue = manager.Queue()
                stopped = Stopped()
                procs = []
                for i, args in enumerate(self.fmllr_rescore_arguments()):

Answer:                     function = FmllrRescoreFunction(args)