Import initial model

am/jit_script.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b64964bad8c24fe48f5d9c0ffe98c4787495ea991a186f00b059fc3fa549c9
+size 264940286

decode.py

@@ -0,0 +1,695 @@
+#!/usr/bin/env python3
+# Copyright 2021-2023 Xiaomi Corporation (Author: Fangjun Kuang, Zengwei Yao)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import argparse
+import logging
+import math
+import warnings
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple, Union
+
+import kaldifeat
+import sentencepiece as spm
+import torch
+import torchaudio
+
+from torch.nn.utils.rnn import pad_sequence
+
+from icefall import NgramLm, NgramLmStateCost
+from icefall.decode import Nbest, one_best_decoding
+from icefall.lm_wrapper import LmScorer
+from icefall.rnn_lm.model import RnnLmModel
+from icefall.transformer_lm.model import TransformerLM
+from icefall.utils import AttributeDict
+from icefall.lexicon import Lexicon
+
+import k2
+
+
+def read_sound_files(
+    filenames: List[str], expected_sample_rate: float = 16000
+) -> List[torch.Tensor]:
+    """Read a list of sound files into a list 1-D float32 torch tensors.
+    Args:
+      filenames:
+        A list of sound filenames.
+      expected_sample_rate:
+        The expected sample rate of the sound files.
+    Returns:
+      Return a list of 1-D float32 torch tensors.
+    """
+    ans = []
+    for f in filenames:
+        wave, sample_rate = torchaudio.load(f)
+        resampler = torchaudio.transforms.Resample(sample_rate, 16_000)
+        wav = resampler(wave[0])
+        ans.append(wav)
+    return ans
+
+@dataclass
+class Hypothesis:
+    # The predicted tokens so far.
+    # Newly predicted tokens are appended to `ys`.
+    ys: List[int]
+
+    # The log prob of ys.
+    # It contains only one entry.
+    log_prob: torch.Tensor
+
+    # timestamp[i] is the frame index after subsampling
+    # on which ys[i] is decoded
+    timestamp: List[int] = field(default_factory=list)
+
+    # the lm score for next token given the current ys
+    lm_score: Optional[torch.Tensor] = None
+
+    # the RNNLM states (h and c in LSTM)
+    state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
+
+    # N-gram LM state
+    state_cost: Optional[NgramLmStateCost] = None
+
+    @property
+    def key(self) -> str:
+        """Return a string representation of self.ys"""
+        return "_".join(map(str, self.ys))
+
+
+class HypothesisList(object):
+    def __init__(self, data: Optional[Dict[str, Hypothesis]] = None) -> None:
+        """
+        Args:
+          data:
+            A dict of Hypotheses. Its key is its `value.key`.
+        """
+        if data is None:
+            self._data = {}
+        else:
+            self._data = data
+
+    @property
+    def data(self) -> Dict[str, Hypothesis]:
+        return self._data
+
+    def add(self, hyp: Hypothesis) -> None:
+        """Add a Hypothesis to `self`.
+
+        If `hyp` already exists in `self`, its probability is updated using
+        `log-sum-exp` with the existed one.
+
+        Args:
+          hyp:
+            The hypothesis to be added.
+        """
+        key = hyp.key
+        if key in self:
+            old_hyp = self._data[key]  # shallow copy
+            torch.logaddexp(old_hyp.log_prob, hyp.log_prob, out=old_hyp.log_prob)
+        else:
+            self._data[key] = hyp
+
+    def get_most_probable(self, length_norm: bool = False) -> Hypothesis:
+        """Get the most probable hypothesis, i.e., the one with
+        the largest `log_prob`.
+
+        Args:
+          length_norm:
+            If True, the `log_prob` of a hypothesis is normalized by the
+            number of tokens in it.
+        Returns:
+          Return the hypothesis that has the largest `log_prob`.
+        """
+        if length_norm:
+            return max(self._data.values(), key=lambda hyp: hyp.log_prob / len(hyp.ys))
+        else:
+            return max(self._data.values(), key=lambda hyp: hyp.log_prob)
+
+    def remove(self, hyp: Hypothesis) -> None:
+        """Remove a given hypothesis.
+
+        Caution:
+          `self` is modified **in-place**.
+
+        Args:
+          hyp:
+            The hypothesis to be removed from `self`.
+            Note: It must be contained in `self`. Otherwise,
+            an exception is raised.
+        """
+        key = hyp.key
+        assert key in self, f"{key} does not exist"
+        del self._data[key]
+
+    def filter(self, threshold: torch.Tensor) -> "HypothesisList":
+        """Remove all Hypotheses whose log_prob is less than threshold.
+
+        Caution:
+          `self` is not modified. Instead, a new HypothesisList is returned.
+
+        Returns:
+          Return a new HypothesisList containing all hypotheses from `self`
+          with `log_prob` being greater than the given `threshold`.
+        """
+        ans = HypothesisList()
+        for _, hyp in self._data.items():
+            if hyp.log_prob > threshold:
+                ans.add(hyp)  # shallow copy
+        return ans
+
+    def topk(self, k: int, length_norm: bool = False) -> "HypothesisList":
+        """Return the top-k hypothesis.
+
+        Args:
+          length_norm:
+            If True, the `log_prob` of a hypothesis is normalized by the
+            number of tokens in it.
+        """
+        hyps = list(self._data.items())
+
+        if length_norm:
+            hyps = sorted(
+                hyps, key=lambda h: h[1].log_prob / len(h[1].ys), reverse=True
+            )[:k]
+        else:
+            hyps = sorted(hyps, key=lambda h: h[1].log_prob, reverse=True)[:k]
+
+        ans = HypothesisList(dict(hyps))
+        return ans
+
+    def __contains__(self, key: str):
+        return key in self._data
+
+    def __iter__(self):
+        return iter(self._data.values())
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __str__(self) -> str:
+        s = []
+        for key in self:
+            s.append(key)
+        return ", ".join(s)
+
+
+def get_hyps_shape(hyps: List[HypothesisList]) -> k2.RaggedShape:
+    """Return a ragged shape with axes [utt][num_hyps].
+
+    Args:
+      hyps:
+        len(hyps) == batch_size. It contains the current hypothesis for
+        each utterance in the batch.
+    Returns:
+      Return a ragged shape with 2 axes [utt][num_hyps]. Note that
+      the shape is on CPU.
+    """
+    num_hyps = [len(h) for h in hyps]
+
+    # torch.cumsum() is inclusive sum, so we put a 0 at the beginning
+    # to get exclusive sum later.
+    num_hyps.insert(0, 0)
+
+    num_hyps = torch.tensor(num_hyps)
+    row_splits = torch.cumsum(num_hyps, dim=0, dtype=torch.int32)
+    ans = k2.ragged.create_ragged_shape2(
+        row_splits=row_splits, cached_tot_size=row_splits[-1].item()
+    )
+    return ans
+
+
+def modified_beam_search_LODR(
+    model,
+    encoder_out: torch.Tensor,
+    encoder_out_lens: torch.Tensor,
+    LODR_lm: NgramLm,
+    LODR_lm_scale: float,
+    LM: LmScorer,
+    beam: int = 4,
+) -> List[List[int]]:
+    """This function implements LODR (https://arxiv.org/abs/2203.16776) with
+    `modified_beam_search`. It uses a bi-gram language model as the estimate
+    of the internal language model and subtracts its score during shallow fusion
+    with an external language model. This implementation uses a RNNLM as the
+    external language model.
+
+    Args:
+        model (Transducer):
+            The transducer model
+        encoder_out (torch.Tensor):
+            Encoder output in (N,T,C)
+        encoder_out_lens (torch.Tensor):
+            A 1-D tensor of shape (N,), containing the number of
+            valid frames in encoder_out before padding.
+        LODR_lm:
+            A low order n-gram LM, whose score will be subtracted during shallow fusion
+        LODR_lm_scale:
+            The scale of the LODR_lm
+        LM:
+            A neural net LM, e.g an RNNLM or transformer LM
+        beam (int, optional):
+            Beam size. Defaults to 4.
+
+    Returns:
+      Return a list-of-list of token IDs. ans[i] is the decoding results
+      for the i-th utterance.
+
+    """
+    assert encoder_out.ndim == 3, encoder_out.shape
+    assert encoder_out.size(0) >= 1, encoder_out.size(0)
+    assert LM is not None
+    lm_scale = LM.lm_scale
+
+    packed_encoder_out = torch.nn.utils.rnn.pack_padded_sequence(
+        input=encoder_out,
+        lengths=encoder_out_lens.cpu(),
+        batch_first=True,
+        enforce_sorted=False,
+    )
+
+    blank_id = model.decoder.blank_id
+    sos_id = getattr(LM, "sos_id", 1)
+    unk_id = getattr(model, "unk_id", blank_id)
+    context_size = model.decoder.context_size
+    device = next(model.parameters()).device
+
+    batch_size_list = packed_encoder_out.batch_sizes.tolist()
+    N = encoder_out.size(0)
+    assert torch.all(encoder_out_lens > 0), encoder_out_lens
+    assert N == batch_size_list[0], (N, batch_size_list)
+
+    # get initial lm score and lm state by scoring the "sos" token
+    sos_token = torch.tensor([[sos_id]]).to(torch.int64).to(device)
+    lens = torch.tensor([1]).to(device)
+    init_score, init_states = LM.score_token(sos_token, lens)
+
+    B = [HypothesisList() for _ in range(N)]
+    for i in range(N):
+        B[i].add(
+            Hypothesis(
+                ys=[blank_id] * context_size,
+                log_prob=torch.zeros(1, dtype=torch.float32, device=device),
+                state=init_states,  # state of the NN LM
+                lm_score=init_score.reshape(-1),
+                state_cost=NgramLmStateCost(
+                    LODR_lm
+                ),  # state of the source domain ngram
+            )
+        )
+
+    encoder_out = model.joiner.encoder_proj(packed_encoder_out.data)
+
+    offset = 0
+    finalized_B = []
+    for batch_size in batch_size_list:
+        start = offset
+        end = offset + batch_size
+        current_encoder_out = encoder_out.data[start:end]  # get batch
+        current_encoder_out = current_encoder_out.unsqueeze(1).unsqueeze(1)
+        # current_encoder_out's shape is (batch_size, 1, 1, encoder_out_dim)
+        offset = end
+
+        finalized_B = B[batch_size:] + finalized_B
+        B = B[:batch_size]
+
+        hyps_shape = get_hyps_shape(B).to(device)
+
+        A = [list(b) for b in B]
+        B = [HypothesisList() for _ in range(batch_size)]
+
+        ys_log_probs = torch.cat(
+            [hyp.log_prob.reshape(1, 1) for hyps in A for hyp in hyps]
+        )
+
+        decoder_input = torch.tensor(
+            [hyp.ys[-context_size:] for hyps in A for hyp in hyps],
+            device=device,
+            dtype=torch.int64,
+        )  # (num_hyps, context_size)
+
+        decoder_out = model.decoder(decoder_input, need_pad=False).unsqueeze(1)
+        decoder_out = model.joiner.decoder_proj(decoder_out)
+
+        current_encoder_out = torch.index_select(
+            current_encoder_out,
+            dim=0,
+            index=hyps_shape.row_ids(1).to(torch.int64),
+        )  # (num_hyps, 1, 1, encoder_out_dim)
+
+        logits = model.joiner(
+            current_encoder_out,
+            decoder_out,
+            project_input=False,
+        )  # (num_hyps, 1, 1, vocab_size)
+
+        logits = logits.squeeze(1).squeeze(1)  # (num_hyps, vocab_size)
+
+        log_probs = logits.log_softmax(dim=-1)  # (num_hyps, vocab_size)
+
+        log_probs.add_(ys_log_probs)
+
+        vocab_size = log_probs.size(-1)
+
+        log_probs = log_probs.reshape(-1)
+
+        row_splits = hyps_shape.row_splits(1) * vocab_size
+        log_probs_shape = k2.ragged.create_ragged_shape2(
+            row_splits=row_splits, cached_tot_size=log_probs.numel()
+        )
+        ragged_log_probs = k2.RaggedTensor(shape=log_probs_shape, value=log_probs)
+        """
+        for all hyps with a non-blank new token, score this token.
+        It is a little confusing here because this for-loop
+        looks very similar to the one below. Here, we go through all
+        top-k tokens and only add the non-blanks ones to the token_list.
+        LM will score those tokens given the LM states. Note that
+        the variable `scores` is the LM score after seeing the new
+        non-blank token.
+        """
+        token_list = []
+        hs = []
+        cs = []
+        for i in range(batch_size):
+            topk_log_probs, topk_indexes = ragged_log_probs[i].topk(beam)
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                topk_hyp_indexes = (topk_indexes // vocab_size).tolist()
+                topk_token_indexes = (topk_indexes % vocab_size).tolist()
+            for k in range(len(topk_hyp_indexes)):
+                hyp_idx = topk_hyp_indexes[k]
+                hyp = A[i][hyp_idx]
+
+                new_token = topk_token_indexes[k]
+                if new_token not in (blank_id, unk_id):
+                    if LM.lm_type == "rnn":
+                        token_list.append([new_token])
+                        # store the LSTM states
+                        hs.append(hyp.state[0])
+                        cs.append(hyp.state[1])
+                    else:
+                        # for transformer LM
+                        token_list.append(
+                            [sos_id] + hyp.ys[context_size:] + [new_token]
+                        )
+
+        # forward NN LM to get new states and scores
+        if len(token_list) != 0:
+            x_lens = torch.tensor([len(tokens) for tokens in token_list]).to(device)
+            if LM.lm_type == "rnn":
+                tokens_to_score = (
+                    torch.tensor(token_list).to(torch.int64).to(device).reshape(-1, 1)
+                )
+                hs = torch.cat(hs, dim=1).to(device)
+                cs = torch.cat(cs, dim=1).to(device)
+                state = (hs, cs)
+            else:
+                # for transformer LM
+                tokens_list = [torch.tensor(tokens) for tokens in token_list]
+                tokens_to_score = (
+                    torch.nn.utils.rnn.pad_sequence(
+                        tokens_list, batch_first=True, padding_value=0.0
+                    )
+                    .to(device)
+                    .to(torch.int64)
+                )
+
+                state = None
+
+            scores, lm_states = LM.score_token(tokens_to_score, x_lens, state)
+
+        count = 0  # index, used to locate score and lm states
+        for i in range(batch_size):
+            topk_log_probs, topk_indexes = ragged_log_probs[i].topk(beam)
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                topk_hyp_indexes = (topk_indexes // vocab_size).tolist()
+                topk_token_indexes = (topk_indexes % vocab_size).tolist()
+
+            for k in range(len(topk_hyp_indexes)):
+                hyp_idx = topk_hyp_indexes[k]
+                hyp = A[i][hyp_idx]
+
+                ys = hyp.ys[:]
+
+                # current score of hyp
+                lm_score = hyp.lm_score
+                state = hyp.state
+
+                hyp_log_prob = topk_log_probs[k]  # get score of current hyp
+                new_token = topk_token_indexes[k]
+                if new_token not in (blank_id, unk_id):
+
+                    ys.append(new_token)
+                    state_cost = hyp.state_cost.forward_one_step(new_token)
+
+                    # calculate the score of the latest token
+                    current_ngram_score = state_cost.lm_score - hyp.state_cost.lm_score
+
+                    assert current_ngram_score <= 0.0, (
+                        state_cost.lm_score,
+                        hyp.state_cost.lm_score,
+                    )
+                    # score = score + TDLM_score - LODR_score
+                    # LODR_LM_scale should be a negative number here
+                    hyp_log_prob += (
+                        lm_score[new_token] * lm_scale
+                        + LODR_lm_scale * current_ngram_score
+                    )  # add the lm score
+
+                    lm_score = scores[count]
+                    if LM.lm_type == "rnn":
+                        state = (
+                            lm_states[0][:, count, :].unsqueeze(1),
+                            lm_states[1][:, count, :].unsqueeze(1),
+                        )
+                    count += 1
+                else:
+                    state_cost = hyp.state_cost
+
+                new_hyp = Hypothesis(
+                    ys=ys,
+                    log_prob=hyp_log_prob,
+                    state=state,
+                    lm_score=lm_score,
+                    state_cost=state_cost,
+                )
+                B[i].add(new_hyp)
+
+    B = B + finalized_B
+    best_hyps = [b.get_most_probable(length_norm=True) for b in B]
+
+    sorted_ans = [h.ys[context_size:] for h in best_hyps]
+    ans = []
+    unsorted_indices = packed_encoder_out.unsorted_indices.tolist()
+    for i in range(N):
+        ans.append(sorted_ans[unsorted_indices[i]])
+
+    return ans
+
+
+def greedy_search(
+    model: torch.jit.ScriptModule,
+    encoder_out: torch.Tensor,
+    encoder_out_lens: torch.Tensor,
+) -> List[List[int]]:
+    """Greedy search in batch mode. It hardcodes --max-sym-per-frame=1.
+    Args:
+      model:
+        The transducer model.
+      encoder_out:
+        A 3-D tensor of shape (N, T, C)
+      encoder_out_lens:
+        A 1-D tensor of shape (N,).
+    Returns:
+      Return the decoded results for each utterance.
+    """
+    assert encoder_out.ndim == 3
+    assert encoder_out.size(0) >= 1, encoder_out.size(0)
+
+    packed_encoder_out = torch.nn.utils.rnn.pack_padded_sequence(
+        input=encoder_out,
+        lengths=encoder_out_lens.cpu(),
+        batch_first=True,
+        enforce_sorted=False,
+    )
+
+    device = encoder_out.device
+    blank_id = 0  # hard-code to 0
+
+    batch_size_list = packed_encoder_out.batch_sizes.tolist()
+    N = encoder_out.size(0)
+
+    assert torch.all(encoder_out_lens > 0), encoder_out_lens
+    assert N == batch_size_list[0], (N, batch_size_list)
+
+    context_size = model.decoder.context_size
+    hyps = [[blank_id] * context_size for _ in range(N)]
+
+    decoder_input = torch.tensor(
+        hyps,
+        device=device,
+        dtype=torch.int64,
+    )  # (N, context_size)
+
+    decoder_out = model.decoder(
+        decoder_input,
+        need_pad=torch.tensor([False]),
+    ).squeeze(1)
+
+    offset = 0
+    for batch_size in batch_size_list:
+        start = offset
+        end = offset + batch_size
+        current_encoder_out = packed_encoder_out.data[start:end]
+        current_encoder_out = current_encoder_out
+        # current_encoder_out's shape: (batch_size, encoder_out_dim)
+        offset = end
+
+        decoder_out = decoder_out[:batch_size]
+
+        logits = model.joiner(
+            current_encoder_out,
+            decoder_out,
+        )
+        # logits'shape (batch_size, vocab_size)
+
+        assert logits.ndim == 2, logits.shape
+        y = logits.argmax(dim=1).tolist()
+        emitted = False
+        for i, v in enumerate(y):
+            if v != blank_id:
+                hyps[i].append(v)
+                emitted = True
+        if emitted:
+            # update decoder output
+            decoder_input = [h[-context_size:] for h in hyps[:batch_size]]
+            decoder_input = torch.tensor(
+                decoder_input,
+                device=device,
+                dtype=torch.int64,
+            )
+            decoder_out = model.decoder(
+                decoder_input,
+                need_pad=torch.tensor([False]),
+            )
+            decoder_out = decoder_out.squeeze(1)
+
+    sorted_ans = [h[context_size:] for h in hyps]
+    ans = []
+    unsorted_indices = packed_encoder_out.unsorted_indices.tolist()
+    for i in range(N):
+        ans.append(sorted_ans[unsorted_indices[i]])
+
+    return ans
+
+
+@torch.no_grad()
+def main():
+    torch.set_num_threads(8)
+
+    device = torch.device("cpu")
+    if torch.cuda.is_available():
+        device = torch.device("cuda", 0)
+
+    model = torch.jit.load("am/jit_script.pt")
+
+    model.eval()
+
+    model.to(device)
+
+    sp = spm.SentencePieceProcessor()
+    sp.load("lang/bpe.model")
+
+    opts = kaldifeat.FbankOptions()
+    opts.device = device
+    opts.frame_opts.dither = 0
+    opts.frame_opts.snip_edges = False
+    opts.frame_opts.samp_freq = 16000
+    opts.mel_opts.num_bins = 80
+
+    fbank = kaldifeat.Fbank(opts)
+
+    all_filenames = sys.argv[1:]
+
+    params = AttributeDict()
+    params.vocab_size = 500
+    params.rnn_lm_embedding_dim = 2048
+    params.rnn_lm_hidden_dim = 2048
+    params.rnn_lm_num_layers = 3
+    params.rnn_lm_tie_weights = True
+    params.lm_epoch = 99
+    params.lm_exp_dir = "lm"
+    params.lm_avg = 1
+
+    LM = LmScorer(
+        lm_type="rnn",
+        params=params,
+        device=device,
+        lm_scale=0.2,
+    )
+    LM.to(device)
+    LM.eval()
+
+    ngram_lm = NgramLm(
+        "lm/2gram.fst.txt",
+        backoff_id=500,
+        is_binary=False,
+    )
+    ngram_lm_scale = -0.1
+
+    for i in range(0, len(all_filenames), 16):
+        filenames = all_filenames[i:i+16]
+        waves = read_sound_files(
+            filenames=filenames,
+        )
+        waves = [w.to(device) for w in waves]
+
+        features = fbank(waves)
+        feature_lengths = [f.size(0) for f in features]
+
+        features = pad_sequence(
+            features,
+            batch_first=True,
+            padding_value=math.log(1e-10),
+        )
+
+        feature_lengths = torch.tensor(feature_lengths, device=device)
+
+        encoder_out, encoder_out_lens = model.encoder(
+            features=features,
+            feature_lengths=feature_lengths,
+        )
+
+        hyps = modified_beam_search_LODR(
+            model=model,
+            encoder_out=encoder_out,
+            encoder_out_lens=encoder_out_lens,
+            beam=20,
+            LODR_lm=ngram_lm,
+            LODR_lm_scale=ngram_lm_scale,
+            LM=LM,
+        )
+
+        for f, hyp in zip(filenames, hyps):
+            words = sp.decode(hyp)
+            print(f"{f.split('/')[-1][0:-4]} {words}")
+
+if __name__ == "__main__":
+    formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
+
+    logging.basicConfig(format=formatter, level=logging.INFO)
+    main()

lang/bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7a756aeb3550417d6b2ed3efde9a7aa3eea54787d4eac011e9cce6090c9c64a
+size 246184

lang/tokens.txt

@@ -0,0 +1,502 @@
+<blk> 0
+<sos/eos> 1
+<unk> 2
+е 3
+▁с 4
+т 5
+▁в 6
+▁ 7
+с 8
+н 9
+м 10
+▁и 11
+и 12
+р 13
+▁на 14
+▁по 15
+д 16
+в 17
+у 18
+л 19
+▁не 20
+к 21
+й 22
+я 23
+а 24
+ра 25
+▁у 26
+ли 27
+но 28
+ла 29
+ка 30
+х 31
+ч 32
+о 33
+▁за 34
+з 35
+ю 36
+ло 37
+ни 38
+▁а 39
+то 40
+ст 41
+ва 42
+г 43
+ы 44
+▁что 45
+ш 46
+ть 47
+ь 48
+ль 49
+▁к 50
+▁я 51
+го 52
+▁это 53
+ж 54
+на 55
+ро 56
+▁о 57
+ти 58
+▁то 59
+ле 60
+во 61
+ре 62
+ки 63
+п 64
+да 65
+▁вы 66
+б 67
+та 68
+▁до 69
+ри 70
+▁при 71
+▁как 72
+▁от 73
+▁но 74
+▁про 75
+▁так 76
+ко 77
+▁да 78
+▁д 79
+ви 80
+не 81
+те 82
+ет 83
+ру 84
+▁со 85
+▁об 86
+ди 87
+▁все 88
+▁ко 89
+ми 90
+ой 91
+ве 92
+▁мо 93
+чи 94
+ля 95
+- 96
+▁он 97
+ё 98
+ку 99
+ов 100
+ом 101
+до 102
+че 103
+▁п 104
+ма 105
+ча 106
+▁го 107
+ту 108
+де 109
+ся 110
+▁ма 111
+лу 112
+же 113
+ц 114
+▁т 115
+▁г 116
+▁м 117
+▁из 118
+ем 119
+ан 120
+▁вот 121
+▁во 122
+▁раз 123
+жи 124
+ста 125
+по 126
+ши 127
+ный 128
+▁есть 129
+▁б 130
+ше 131
+▁ли 132
+за 133
+▁з 134
+бо 135
+га 136
+▁ка 137
+▁мы 138
+мо 139
+▁де 140
+сти 141
+▁те 142
+ну 143
+▁под 144
+тор 145
+ить 146
+▁бы 147
+▁ни 148
+▁ф 149
+▁э 150
+▁ре 151
+ные 152
+ать 153
+ры 154
+мен 155
+▁мне 156
+ение 157
+ей 158
+ня 159
+па 160
+▁же 161
+му 162
+пе 163
+ения 164
+ду 165
+ги 166
+ф 167
+сь 168
+об 169
+ны 170
+ется 171
+ты 172
+▁пере 173
+со 174
+ую 175
+ал 176
+▁его 177
+ого 178
+лю 179
+це 180
+▁ну 181
+вер 182
+ной 183
+ци 184
+жа 185
+им 186
+щи 187
+▁па 188
+би 189
+ят 190
+▁ро 191
+▁если 192
+ного 193
+ёт 194
+ск 195
+ных 196
+ход 197
+щ 198
+ще 199
+ая 200
+ров 201
+▁меня 202
+▁ш 203
+▁она 204
+жд 205
+▁ты 206
+бе 207
+▁ку 208
+▁рас 209
+мер 210
+▁для 211
+ца 212
+ря 213
+ень 214
+▁бо 215
+▁бе 216
+си 217
+тель 218
+ная 219
+ным 220
+он 221
+▁ещё 222
+▁будет 223
+▁хо 224
+вы 225
+▁ми 226
+са 227
+▁там 228
+лы 229
+ша 230
+▁ле 231
+▁они 232
+пу 233
+зна 234
+нов 235
+пи 236
+ое 237
+ба 238
+ил 239
+дел 240
+▁был 241
+ско 242
+▁ба 243
+ите 244
+▁друг 245
+▁че 246
+▁вам 247
+▁х 248
+ый 249
+▁вид 250
+лов 251
+▁час 252
+▁было 253
+▁только 254
+▁вас 255
+ке 256
+тер 257
+э 258
+▁или 259
+▁когда 260
+▁сто 261
+▁уже 262
+▁нас 263
+вать 264
+▁пред 265
+▁может 266
+▁включи 267
+▁очень 268
+хо 269
+лись 270
+ха 271
+▁ра 272
+сть 273
+ство 274
+▁пре 275
+ное 276
+ков 277
+▁кон 278
+▁ис 279
+ами 280
+▁оп 281
+▁е 282
+▁тебя 283
+лась 284
+▁сам 285
+ания 286
+раз 287
+ник 288
+зы 289
+▁фильм 290
+▁один 291
+▁эти 292
+ним 293
+▁чтобы 294
+ание 295
+ции 296
+чно 297
+▁сейчас 298
+▁бу 299
+▁нет 300
+ком 301
+▁просто 302
+ём 303
+ён 304
+лось 305
+фи 306
+▁три 307
+из 308
+аться 309
+гу 310
+смотр 311
+▁воз 312
+вор 313
+▁ком 314
+▁пер 315
+зи 316
+▁ж 317
+▁ос 318
+▁можно 319
+▁её 320
+пол 321
+▁два 322
+▁оста 323
+▁вер 324
+нул 325
+▁сер 326
+ент 327
+▁люб 328
+ых 329
+ную 330
+пис 331
+▁чем 332
+▁после 333
+▁сказал 334
+▁пу 335
+тов 336
+айте 337
+▁му 338
+▁человек 339
+▁кар 340
+кой 341
+ешь 342
+ится 343
+▁сезон 344
+ают 345
+бы 346
+▁тв 347
+ился 348
+ально 349
+▁могу 350
+▁мест 351
+▁две 352
+тро 353
+пра 354
+▁боль 355
+ался 356
+▁где 357
+сси 358
+▁пи 359
+ща 360
+форм 361
+▁потому 362
+▁быть 363
+▁двадцать 364
+▁ваш 365
+ёл 366
+про 367
+ность 368
+▁без 369
+▁дела 370
+▁та 371
+ъ 372
+ически 373
+▁пе 374
+▁которые 375
+▁пожалуйста 376
+▁само 377
+▁одно 378
+▁работ 379
+▁пять 380
+ский 381
+▁прав 382
+▁время 383
+▁даже 384
+▁смотрешке 385
+ция 386
+▁здесь 387
+▁четыре 388
+ской 389
+ственно 390
+▁этого 391
+▁гр 392
+▁нужно 393
+▁билет 394
+▁рук 395
+ности 396
+иться 397
+▁чи 398
+▁какой 399
+▁тре 400
+▁ему 401
+ются 402
+ительно 403
+▁стан 404
+▁который 405
+▁свои 406
+▁семь 407
+▁стар 408
+▁себя 409
+▁хочу 410
+▁сколько 411
+▁теперь 412
+▁буду 413
+▁какие 414
+▁салют 415
+▁больше 416
+▁кино 417
+▁потом 418
+▁восемь 419
+▁ничего 420
+▁след 421
+▁минут 422
+ская 423
+▁поезд 424
+▁сериал 425
+▁вопрос 426
+▁спасибо 427
+▁канал 428
+▁говорит 429
+▁хорошо 430
+▁жив 431
+▁девять 432
+▁через 433
+▁шесть 434
+▁конечно 435
+▁тогда 436
+▁покажи 437
+▁стал 438
+▁вообще 439
+ывает 440
+▁интерес 441
+▁себе 442
+▁крас 443
+▁тридцать 444
+▁сегодня 445
+▁действ 446
+▁сторон 447
+▁чу 448
+ении 449
+▁почему 450
+▁более 451
+▁поэтому 452
+▁десят 453
+▁отправ 454
+▁знаю 455
+ских 456
+▁тысяч 457
+▁всё 458
+▁можешь 459
+▁значит 460
+▁против 461
+▁ответ 462
+▁найди 463
+▁люди 464
+▁одна 465
+▁начал 466
+▁серия 467
+▁между 468
+▁свет 469
+▁именно 470
+▁телефон 471
+нибудь 472
+▁пятьдесят 473
+очка 474
+▁времени 475
+▁цел 476
+▁возможно 477
+▁сказать 478
+▁глаза 479
+▁называ 480
+▁хотел 481
+▁сорок 482
+▁поставь 483
+▁первый 484
+▁связ 485
+▁которая 486
+▁электр 487
+▁например 488
+▁ведь 489
+▁сделать 490
+▁говорил 491
+▁благо 492
+ировать 493
+▁посмотреть 494
+▁всегда 495
+▁несколько 496
+▁людей 497
+▁случае 498
+▁гарри 499
+#0 500
+#1 501

lang/unigram_500.vocab

@@ -0,0 +1,500 @@
+<blk>	0
+<sos/eos>	0
+<unk>	0
+е	-3.72421
+▁с	-4.00711
+т	-4.08971
+▁в	-4.14234
+▁	-4.14364
+с	-4.15149
+н	-4.16546
+м	-4.20176
+▁и	-4.21221
+и	-4.29997
+р	-4.37977
+▁на	-4.3817
+▁по	-4.38887
+д	-4.41525
+в	-4.41704
+у	-4.43597
+л	-4.53253
+▁не	-4.56287
+к	-4.58558
+й	-4.60495
+я	-4.64027
+а	-4.64466
+ра	-4.75414
+▁у	-4.7655
+ли	-4.81635
+но	-4.86806
+ла	-4.87179
+ка	-4.92484
+х	-4.9503
+ч	-4.95321
+о	-4.98385
+▁за	-5.05821
+з	-5.05928
+ю	-5.08551
+ло	-5.11068
+ни	-5.1306
+▁а	-5.13288
+то	-5.13924
+ст	-5.14875
+ва	-5.15182
+г	-5.16538
+ы	-5.18352
+▁что	-5.22316
+ш	-5.22707
+ть	-5.23793
+ь	-5.2509
+ль	-5.25799
+▁к	-5.26308
+▁я	-5.26591
+го	-5.27411
+▁это	-5.29806
+ж	-5.3083
+на	-5.30963
+ро	-5.3109
+▁о	-5.34496
+ти	-5.3955
+▁то	-5.40496
+ле	-5.41254
+во	-5.4165
+ре	-5.4188
+ки	-5.47916
+п	-5.48377
+да	-5.52795
+▁вы	-5.52823
+б	-5.52901
+та	-5.54602
+▁до	-5.55857
+ри	-5.56377
+▁при	-5.58046
+▁как	-5.58207
+▁от	-5.63345
+▁но	-5.63407
+▁про	-5.63691
+▁так	-5.6499
+ко	-5.65805
+▁да	-5.67491
+▁д	-5.67721
+ви	-5.68739
+не	-5.6981
+те	-5.74268
+ет	-5.77888
+ру	-5.78523
+▁со	-5.80155
+▁об	-5.80862
+ди	-5.85919
+▁все	-5.88436
+▁ко	-5.90038
+ми	-5.91224
+ой	-5.91948
+ве	-5.92044
+▁мо	-5.9268
+чи	-5.93594
+ля	-5.94037
+-	-5.95904
+▁он	-5.97629
+ё	-5.99857
+ку	-6.00185
+ов	-6.01007
+ом	-6.0288
+до	-6.04131
+че	-6.0471
+▁п	-6.05797
+ма	-6.06525
+ча	-6.07659
+▁го	-6.08481
+ту	-6.0907
+де	-6.09127
+ся	-6.10754
+▁ма	-6.12254
+лу	-6.12261
+же	-6.15441
+ц	-6.17019
+▁т	-6.17707
+▁г	-6.18064
+▁м	-6.187
+▁из	-6.18745
+ем	-6.18764
+ан	-6.1898
+▁вот	-6.20622
+▁во	-6.21312
+▁раз	-6.21391
+жи	-6.21908
+ста	-6.2236
+по	-6.23241
+ши	-6.23353
+ный	-6.23429
+▁есть	-6.25535
+▁б	-6.25862
+ше	-6.26684
+▁ли	-6.2692
+за	-6.27292
+▁з	-6.27702
+бо	-6.29133
+га	-6.29361
+▁ка	-6.30692
+▁мы	-6.30719
+мо	-6.30974
+▁де	-6.31045
+сти	-6.32652
+▁те	-6.34079
+ну	-6.36912
+▁под	-6.37186
+тор	-6.37209
+ить	-6.37813
+▁бы	-6.38675
+▁ни	-6.38974
+▁ф	-6.395
+▁э	-6.39689
+▁ре	-6.4069
+ные	-6.40992
+ать	-6.41185
+ры	-6.41376
+мен	-6.42197
+▁мне	-6.42556
+ение	-6.43243
+ей	-6.44644
+ня	-6.45298
+па	-6.45865
+▁же	-6.46201
+му	-6.48738
+пе	-6.49161
+ения	-6.49489
+ду	-6.50284
+ги	-6.50349
+ф	-6.50672
+сь	-6.50802
+об	-6.51477
+ны	-6.51839
+ется	-6.5246
+ты	-6.52819
+▁пере	-6.53089
+со	-6.53688
+ую	-6.53716
+ал	-6.53947
+▁его	-6.54683
+ого	-6.5558
+лю	-6.57673
+це	-6.60805
+▁ну	-6.61118
+вер	-6.61359
+ной	-6.61434
+ци	-6.61518
+жа	-6.62145
+им	-6.62822
+щи	-6.6424
+▁па	-6.65636
+би	-6.65941
+ят	-6.67541
+▁ро	-6.69099
+▁если	-6.70234
+ного	-6.71022
+ёт	-6.71453
+ск	-6.71786
+ных	-6.71869
+ход	-6.72616
+щ	-6.7273
+ще	-6.73349
+ая	-6.73537
+ров	-6.75237
+▁меня	-6.75302
+▁ш	-6.75798
+▁она	-6.76858
+жд	-6.77673
+▁ты	-6.77867
+бе	-6.79111
+▁ку	-6.79506
+▁рас	-6.79763
+мер	-6.79882
+▁для	-6.8128
+ца	-6.82069
+ря	-6.83478
+ень	-6.83977
+▁бо	-6.84372
+▁бе	-6.84714
+си	-6.85532
+тель	-6.85941
+ная	-6.86527
+ным	-6.86943
+он	-6.88114
+▁ещё	-6.8825
+▁будет	-6.88757
+▁хо	-6.89723
+вы	-6.89731
+▁ми	-6.90201
+са	-6.90676
+▁там	-6.91816
+лы	-6.92175
+ша	-6.92869
+▁ле	-6.93238
+▁они	-6.9348
+пу	-6.93718
+зна	-6.93926
+нов	-6.94786
+пи	-6.95597
+ое	-6.95774
+ба	-6.9586
+ил	-6.96984
+дел	-6.98119
+▁был	-6.98238
+ско	-6.98854
+▁ба	-6.98951
+ите	-6.99367
+▁друг	-6.9943
+▁че	-6.99802
+▁вам	-7.01029
+▁х	-7.01148
+ый	-7.01826
+▁вид	-7.01874
+лов	-7.02428
+▁час	-7.02801
+▁было	-7.03338
+▁только	-7.03819
+▁вас	-7.04398
+ке	-7.05017
+тер	-7.06037
+э	-7.06244
+▁или	-7.06254
+▁когда	-7.06466
+▁сто	-7.07688
+▁уже	-7.0945
+▁нас	-7.0947
+вать	-7.09646
+▁пред	-7.1066
+▁может	-7.10866
+▁включи	-7.1111
+▁очень	-7.11544
+хо	-7.11665
+лись	-7.12812
+ха	-7.13242
+▁ра	-7.13898
+сть	-7.13935
+ство	-7.14754
+▁пре	-7.15134
+ное	-7.15238
+ков	-7.15586
+▁кон	-7.15816
+▁ис	-7.1604
+ами	-7.16615
+▁оп	-7.18563
+▁е	-7.18591
+▁тебя	-7.18813
+лась	-7.19185
+▁сам	-7.19982
+ания	-7.19995
+раз	-7.20058
+ник	-7.20603
+зы	-7.20843
+▁фильм	-7.21124
+▁один	-7.21558
+▁эти	-7.21559
+ним	-7.21944
+▁чтобы	-7.22289
+ание	-7.23062
+ции	-7.23383
+чно	-7.23714
+▁сейчас	-7.24093
+▁бу	-7.24279
+▁нет	-7.24342
+ком	-7.24996
+▁просто	-7.2506
+ём	-7.2565
+ён	-7.26084
+лось	-7.26338
+фи	-7.26581
+▁три	-7.27362
+из	-7.28159
+аться	-7.2843
+гу	-7.28869
+смотр	-7.2938
+▁воз	-7.30375
+вор	-7.3041
+▁ком	-7.31052
+▁пер	-7.3106
+зи	-7.3194
+▁ж	-7.32187
+▁ос	-7.3302
+▁можно	-7.3382
+▁её	-7.33991
+пол	-7.34408
+▁два	-7.34513
+▁оста	-7.3556
+▁вер	-7.36032
+нул	-7.36142
+▁сер	-7.36736
+ент	-7.37146
+▁люб	-7.37199
+ых	-7.37646
+ную	-7.37825
+пис	-7.37833
+▁чем	-7.38971
+▁после	-7.39552
+▁сказал	-7.3956
+▁пу	-7.39903
+тов	-7.39953
+айте	-7.40175
+▁му	-7.41591
+▁человек	-7.42485
+▁кар	-7.427
+кой	-7.42935
+ешь	-7.43681
+ится	-7.44416
+▁сезон	-7.45472
+ают	-7.46059
+бы	-7.46086
+▁тв	-7.46131
+ился	-7.46134
+ально	-7.46924
+▁могу	-7.47201
+▁мест	-7.47594
+▁две	-7.48553
+тро	-7.49256
+пра	-7.49279
+▁боль	-7.49293
+ался	-7.49695
+▁где	-7.50004
+сси	-7.50442
+▁пи	-7.50621
+ща	-7.52085
+форм	-7.54043
+▁потому	-7.54055
+▁быть	-7.55272
+▁двадцать	-7.55597
+▁ваш	-7.5597
+ёл	-7.56701
+про	-7.57489
+ность	-7.57697
+▁без	-7.57787
+▁дела	-7.58268
+▁та	-7.58835
+ъ	-7.59162
+ически	-7.59388
+▁пе	-7.59514
+▁которые	-7.59823
+▁пожалуйста	-7.60476
+▁само	-7.62166
+▁одно	-7.62488
+▁работ	-7.62587
+▁пять	-7.63438
+ский	-7.63578
+▁прав	-7.64631
+▁время	-7.64938
+▁даже	-7.65259
+▁смотрешке	-7.65371
+ция	-7.65981
+▁здесь	-7.6756
+▁четыре	-7.67719
+ской	-7.68614
+ственно	-7.68683
+▁этого	-7.68729
+▁гр	-7.68949
+▁нужно	-7.68976
+▁билет	-7.69119
+▁рук	-7.69124
+ности	-7.70224
+иться	-7.71736
+▁чи	-7.74136
+▁какой	-7.74534
+▁тре	-7.75798
+▁ему	-7.76136
+ются	-7.76285
+ительно	-7.76457
+▁стан	-7.77955
+▁который	-7.78311
+▁свои	-7.79485
+▁семь	-7.80293
+▁стар	-7.82062
+▁себя	-7.829
+▁хочу	-7.82951
+▁сколько	-7.84939
+▁теперь	-7.86712
+▁буду	-7.88652
+▁какие	-7.89257
+▁салют	-7.89721
+▁больше	-7.90363
+▁кино	-7.9076
+▁потом	-7.90802
+▁восемь	-7.92117
+▁ничего	-7.92557
+▁след	-7.93644
+▁минут	-7.93963
+ская	-7.94281
+▁поезд	-7.94517
+▁сериал	-7.95059
+▁вопрос	-7.96073
+▁спасибо	-7.96309
+▁канал	-7.9692
+▁говорит	-7.97006
+▁хорошо	-7.9883
+▁жив	-7.99152
+▁девять	-7.99548
+▁через	-8.00708
+▁шесть	-8.01468
+▁конечно	-8.01557
+▁тогда	-8.03135
+▁покажи	-8.0337
+▁стал	-8.03473
+▁вообще	-8.05943
+ывает	-8.06989
+▁интерес	-8.07217
+▁себе	-8.09189
+▁крас	-8.10057
+▁тридцать	-8.10802
+▁сегодня	-8.11505
+▁действ	-8.12242
+▁сторон	-8.12496
+▁чу	-8.13269
+ении	-8.14634
+▁почему	-8.15047
+▁более	-8.15271
+▁поэтому	-8.16289
+▁десят	-8.16801
+▁отправ	-8.16928
+▁знаю	-8.17321
+ских	-8.18712
+▁тысяч	-8.18932
+▁всё	-8.20101
+▁можешь	-8.2129
+▁значит	-8.214
+▁против	-8.22679
+▁ответ	-8.22794
+▁найди	-8.23782
+▁люди	-8.23857
+▁одна	-8.24067
+▁начал	-8.24422
+▁серия	-8.25009
+▁между	-8.26
+▁свет	-8.2639
+▁именно	-8.2677
+▁телефон	-8.27473
+нибудь	-8.29168
+▁пятьдесят	-8.30322
+очка	-8.30543
+▁времени	-8.31148
+▁цел	-8.33721
+▁возможно	-8.35705
+▁сказать	-8.35962
+▁глаза	-8.37791
+▁называ	-8.3883
+▁хотел	-8.39147
+▁сорок	-8.39892
+▁поставь	-8.41498
+▁первый	-8.41767
+▁связ	-8.4299
+▁которая	-8.43202
+▁электр	-8.43714
+▁например	-8.45837
+▁ведь	-8.45906
+▁сделать	-8.46666
+▁говорил	-8.47077
+▁благо	-8.4716
+ировать	-8.48783
+▁посмотреть	-8.48969
+▁всегда	-8.49153
+▁несколько	-8.49574
+▁людей	-8.49624
+▁случае	-8.5061
+▁гарри	-8.5218

lm/epoch-99.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb6cbccb971a9cfc50c1dc8d2311eee51095ceff68e9bc03810e4176a816feb
+size 406952299