add models

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

GigaAM%20License_NC.pdf

@@ -0,0 +1 @@
+Entry not found

README.md

@@ -0,0 +1,10 @@
+# Introduction
+
+This folder contains scripts for converting models from
+https://github.com/salute-developers/GigaAM
+to sherpa-onnx.
+
+The ASR models are for Russian speech recognition in this folder.
+
+Please see the license of the models at
+https://github.com/salute-developers/GigaAM/blob/main/GigaAM%20License_NC.pdf

decoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bc934fb277e777c055416bff25cba175865163fdc806fda8dd716001b625885
+size 3945078

encoder.int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b8543fbae136cdca47d023deb6d2893dc97e40ba001c3491b84bfef2222dcd9
+size 274772325

export-onnx-rnnt.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# Copyright      2024  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+from typing import Dict
+
+import onnx
+import torch
+import torchaudio
+from nemo.collections.asr.models import EncDecRNNTBPEModel
+from nemo.collections.asr.modules.audio_preprocessing import (
+    AudioToMelSpectrogramPreprocessor as NeMoAudioToMelSpectrogramPreprocessor,
+)
+from nemo.collections.asr.parts.preprocessing.features import (
+    FilterbankFeaturesTA as NeMoFilterbankFeaturesTA,
+)
+from onnxruntime.quantization import QuantType, quantize_dynamic
+
+
+def add_meta_data(filename: str, meta_data: Dict[str, str]):
+    """Add meta data to an ONNX model. It is changed in-place.
+
+    Args:
+      filename:
+        Filename of the ONNX model to be changed.
+      meta_data:
+        Key-value pairs.
+    """
+    model = onnx.load(filename)
+    while len(model.metadata_props):
+        model.metadata_props.pop()
+
+    for key, value in meta_data.items():
+        meta = model.metadata_props.add()
+        meta.key = key
+        meta.value = str(value)
+
+    onnx.save(model, filename)
+
+
+class FilterbankFeaturesTA(NeMoFilterbankFeaturesTA):
+    def __init__(self, mel_scale: str = "htk", wkwargs=None, **kwargs):
+        if "window_size" in kwargs:
+            del kwargs["window_size"]
+        if "window_stride" in kwargs:
+            del kwargs["window_stride"]
+
+        super().__init__(**kwargs)
+
+        self._mel_spec_extractor: torchaudio.transforms.MelSpectrogram = (
+            torchaudio.transforms.MelSpectrogram(
+                sample_rate=self._sample_rate,
+                win_length=self.win_length,
+                hop_length=self.hop_length,
+                n_mels=kwargs["nfilt"],
+                window_fn=self.torch_windows[kwargs["window"]],
+                mel_scale=mel_scale,
+                norm=kwargs["mel_norm"],
+                n_fft=kwargs["n_fft"],
+                f_max=kwargs.get("highfreq", None),
+                f_min=kwargs.get("lowfreq", 0),
+                wkwargs=wkwargs,
+            )
+        )
+
+
+class AudioToMelSpectrogramPreprocessor(NeMoAudioToMelSpectrogramPreprocessor):
+    def __init__(self, mel_scale: str = "htk", **kwargs):
+        super().__init__(**kwargs)
+        kwargs["nfilt"] = kwargs["features"]
+        del kwargs["features"]
+        self.featurizer = (
+            FilterbankFeaturesTA(  # Deprecated arguments; kept for config compatibility
+                mel_scale=mel_scale,
+                **kwargs,
+            )
+        )
+
+
+@torch.no_grad()
+def main():
+    model = EncDecRNNTBPEModel.from_config_file("./rnnt_model_config.yaml")
+    ckpt = torch.load("./rnnt_model_weights.ckpt", map_location="cpu")
+    model.load_state_dict(ckpt, strict=False)
+    model.eval()
+
+    with open("./tokens.txt", "w", encoding="utf-8") as f:
+        for i, s in enumerate(model.joint.vocabulary):
+            f.write(f"{s} {i}\n")
+        f.write(f"<blk> {i+1}\n")
+        print("Saved to tokens.txt")
+
+    model.encoder.export("encoder.onnx")
+    model.decoder.export("decoder.onnx")
+    model.joint.export("joiner.onnx")
+
+    meta_data = {
+        "vocab_size": model.decoder.vocab_size,  # not including the blank
+        "pred_rnn_layers": model.decoder.pred_rnn_layers,
+        "pred_hidden": model.decoder.pred_hidden,
+        "normalize_type": "",
+        "subsampling_factor": 4,
+        "model_type": "EncDecRNNTBPEModel",
+        "version": "1",
+        "model_author": "https://github.com/salute-developers/GigaAM",
+        "license": "https://github.com/salute-developers/GigaAM/blob/main/GigaAM%20License_NC.pdf",
+        "language": "Russian",
+        "is_giga_am": 1,
+    }
+    add_meta_data("encoder.onnx", meta_data)
+
+    quantize_dynamic(
+        model_input="encoder.onnx",
+        model_output="encoder.int8.onnx",
+        weight_type=QuantType.QUInt8,
+    )
+
+
+if __name__ == "__main__":
+    main()

joiner.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb7135757e3957f9f1473236d429bd44145d76209fb11e7b56e01c85e2dcb83
+size 2055583

run-rnnt.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Copyright    2024  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+set -ex
+
+function install_nemo() {
+  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+  python3 get-pip.py
+
+  pip install torch==2.4.0 torchaudio==2.4.0 -f https://download.pytorch.org/whl/torch_stable.html
+
+  pip install -qq wget text-unidecode matplotlib>=3.3.2 onnx onnxruntime pybind11 Cython einops kaldi-native-fbank soundfile librosa
+  pip install -qq ipython
+
+  # sudo apt-get install -q -y sox libsndfile1 ffmpeg python3-pip ipython
+
+  BRANCH='main'
+  python3 -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[asr]
+
+  pip install numpy==1.26.4
+}
+
+function download_files() {
+  # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/rnnt_model_weights.ckpt
+  # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/rnnt_model_config.yaml
+  # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/example.wav
+  # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/long_example.wav
+  # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/tokenizer_all_sets.tar
+
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/rnnt_model_weights.ckpt
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/rnnt_model_config.yaml
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/example.wav
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/long_example.wav
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM%20License_NC.pdf
+  curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/tokenizer_all_sets.tar
+  tar -xf tokenizer_all_sets.tar && rm tokenizer_all_sets.tar
+  ls -lh
+  echo "---"
+  ls -lh tokenizer_all_sets
+  echo "---"
+}
+
+install_nemo
+download_files
+
+python3 ./export-onnx-rnnt.py
+ls -lh
+python3 ./test-onnx-rnnt.py
+rm -v encoder.onnx
+ls -lh

test-onnx-rnnt.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+# Copyright      2024  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+import argparse
+from pathlib import Path
+
+import kaldi_native_fbank as knf
+import librosa
+import numpy as np
+import onnxruntime as ort
+import soundfile as sf
+import torch
+
+
+def create_fbank():
+    opts = knf.FbankOptions()
+    opts.frame_opts.dither = 0
+    opts.frame_opts.remove_dc_offset = False
+    opts.frame_opts.preemph_coeff = 0
+    opts.frame_opts.window_type = "hann"
+
+    # Even though GigaAM uses 400 for fft, here we use 512
+    # since kaldi-native-fbank only support fft for power of 2.
+    opts.frame_opts.round_to_power_of_two = True
+
+    opts.mel_opts.low_freq = 0
+    opts.mel_opts.high_freq = 8000
+    opts.mel_opts.num_bins = 64
+
+    fbank = knf.OnlineFbank(opts)
+    return fbank
+
+
+def compute_features(audio, fbank):
+    assert len(audio.shape) == 1, audio.shape
+    fbank.accept_waveform(16000, audio)
+    ans = []
+    processed = 0
+    while processed < fbank.num_frames_ready:
+        ans.append(np.array(fbank.get_frame(processed)))
+        processed += 1
+    ans = np.stack(ans)
+    return ans
+
+
+def display(sess):
+    print("==========Input==========")
+    for i in sess.get_inputs():
+        print(i)
+    print("==========Output==========")
+    for i in sess.get_outputs():
+        print(i)
+
+
+"""
+==========Input==========
+NodeArg(name='audio_signal', type='tensor(float)', shape=['audio_signal_dynamic_axes_1', 64, 'audio_signal_dynamic_axes_2'])
+NodeArg(name='length', type='tensor(int64)', shape=['length_dynamic_axes_1'])
+==========Output==========
+NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 768, 'outputs_dynamic_axes_2'])
+NodeArg(name='encoded_lengths', type='tensor(int64)', shape=['encoded_lengths_dynamic_axes_1'])
+==========Input==========
+NodeArg(name='targets', type='tensor(int32)', shape=['targets_dynamic_axes_1', 'targets_dynamic_axes_2'])
+NodeArg(name='target_length', type='tensor(int32)', shape=['target_length_dynamic_axes_1'])
+NodeArg(name='states.1', type='tensor(float)', shape=[1, 'states.1_dim_1', 320])
+NodeArg(name='onnx::LSTM_3', type='tensor(float)', shape=[1, 1, 320])
+==========Output==========
+NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 320, 'outputs_dynamic_axes_2'])
+NodeArg(name='prednet_lengths', type='tensor(int32)', shape=['prednet_lengths_dynamic_axes_1'])
+NodeArg(name='states', type='tensor(float)', shape=[1, 'states_dynamic_axes_1', 320])
+NodeArg(name='74', type='tensor(float)', shape=[1, 'states_dynamic_axes_1', 320])
+==========Input==========
+NodeArg(name='encoder_outputs', type='tensor(float)', shape=['encoder_outputs_dynamic_axes_1', 768, 'encoder_outputs_dynamic_axes_2'])
+NodeArg(name='decoder_outputs', type='tensor(float)', shape=['decoder_outputs_dynamic_axes_1', 320, 'decoder_outputs_dynamic_axes_2'])
+==========Output==========
+NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 'outputs_dynamic_axes_2', 'outputs_dynamic_axes_3', 513])
+"""
+
+
+class OnnxModel:
+    def __init__(
+        self,
+        encoder: str,
+        decoder: str,
+        joiner: str,
+    ):
+        self.init_encoder(encoder)
+        display(self.encoder)
+        self.init_decoder(decoder)
+        display(self.decoder)
+        self.init_joiner(joiner)
+        display(self.joiner)
+
+    def init_encoder(self, encoder):
+        session_opts = ort.SessionOptions()
+        session_opts.inter_op_num_threads = 1
+        session_opts.intra_op_num_threads = 1
+
+        self.encoder = ort.InferenceSession(
+            encoder,
+            sess_options=session_opts,
+            providers=["CPUExecutionProvider"],
+        )
+
+        meta = self.encoder.get_modelmeta().custom_metadata_map
+        self.normalize_type = meta["normalize_type"]
+        print(meta)
+
+        self.pred_rnn_layers = int(meta["pred_rnn_layers"])
+        self.pred_hidden = int(meta["pred_hidden"])
+
+    def init_decoder(self, decoder):
+        session_opts = ort.SessionOptions()
+        session_opts.inter_op_num_threads = 1
+        session_opts.intra_op_num_threads = 1
+
+        self.decoder = ort.InferenceSession(
+            decoder,
+            sess_options=session_opts,
+            providers=["CPUExecutionProvider"],
+        )
+
+    def init_joiner(self, joiner):
+        session_opts = ort.SessionOptions()
+        session_opts.inter_op_num_threads = 1
+        session_opts.intra_op_num_threads = 1
+
+        self.joiner = ort.InferenceSession(
+            joiner,
+            sess_options=session_opts,
+            providers=["CPUExecutionProvider"],
+        )
+
+    def get_decoder_state(self):
+        batch_size = 1
+        state0 = torch.zeros(self.pred_rnn_layers, batch_size, self.pred_hidden).numpy()
+        state1 = torch.zeros(self.pred_rnn_layers, batch_size, self.pred_hidden).numpy()
+        return state0, state1
+
+    def run_encoder(self, x: np.ndarray):
+        # x: (T, C)
+        x = torch.from_numpy(x)
+        x = x.t().unsqueeze(0)
+        # x: [1, C, T]
+        x_lens = torch.tensor([x.shape[-1]], dtype=torch.int64)
+
+        (encoder_out, out_len) = self.encoder.run(
+            [
+                self.encoder.get_outputs()[0].name,
+                self.encoder.get_outputs()[1].name,
+            ],
+            {
+                self.encoder.get_inputs()[0].name: x.numpy(),
+                self.encoder.get_inputs()[1].name: x_lens.numpy(),
+            },
+        )
+        # [batch_size, dim, T]
+        return encoder_out
+
+    def run_decoder(
+        self,
+        token: int,
+        state0: np.ndarray,
+        state1: np.ndarray,
+    ):
+        target = torch.tensor([[token]], dtype=torch.int32).numpy()
+        target_len = torch.tensor([1], dtype=torch.int32).numpy()
+
+        (
+            decoder_out,
+            decoder_out_length,
+            state0_next,
+            state1_next,
+        ) = self.decoder.run(
+            [
+                self.decoder.get_outputs()[0].name,
+                self.decoder.get_outputs()[1].name,
+                self.decoder.get_outputs()[2].name,
+                self.decoder.get_outputs()[3].name,
+            ],
+            {
+                self.decoder.get_inputs()[0].name: target,
+                self.decoder.get_inputs()[1].name: target_len,
+                self.decoder.get_inputs()[2].name: state0,
+                self.decoder.get_inputs()[3].name: state1,
+            },
+        )
+        return decoder_out, state0_next, state1_next
+
+    def run_joiner(
+        self,
+        encoder_out: np.ndarray,
+        decoder_out: np.ndarray,
+    ):
+        # encoder_out: [batch_size,  dim, 1]
+        # decoder_out: [batch_size,  dim, 1]
+        logit = self.joiner.run(
+            [
+                self.joiner.get_outputs()[0].name,
+            ],
+            {
+                self.joiner.get_inputs()[0].name: encoder_out,
+                self.joiner.get_inputs()[1].name: decoder_out,
+            },
+        )[0]
+        # logit: [batch_size, 1, 1, vocab_size]
+        return logit
+
+
+def main():
+    model = OnnxModel("encoder.int8.onnx", "decoder.onnx", "joiner.onnx")
+
+    id2token = dict()
+    with open("./tokens.txt", encoding="utf-8") as f:
+        for line in f:
+            t, idx = line.split()
+            id2token[int(idx)] = t
+
+    fbank = create_fbank()
+    audio, sample_rate = sf.read("./example.wav", dtype="float32", always_2d=True)
+    audio = audio[:, 0]  # only use the first channel
+    if sample_rate != 16000:
+        audio = librosa.resample(
+            audio,
+            orig_sr=sample_rate,
+            target_sr=16000,
+        )
+        sample_rate = 16000
+
+    tail_padding = np.zeros(sample_rate * 2)
+
+    audio = np.concatenate([audio, tail_padding])
+
+    blank = len(id2token) - 1
+    ans = [blank]
+    state0, state1 = model.get_decoder_state()
+    decoder_out, state0_next, state1_next = model.run_decoder(ans[-1], state0, state1)
+
+    features = compute_features(audio, fbank)
+    print("audio.shape", audio.shape)
+    print("features.shape", features.shape)
+
+    encoder_out = model.run_encoder(features)
+    # encoder_out:[batch_size, dim, T)
+    for t in range(encoder_out.shape[2]):
+        encoder_out_t = encoder_out[:, :, t : t + 1]
+        logits = model.run_joiner(encoder_out_t, decoder_out)
+        logits = torch.from_numpy(logits)
+        logits = logits.squeeze()
+        idx = torch.argmax(logits, dim=-1).item()
+        if idx != blank:
+            ans.append(idx)
+            state0 = state0_next
+            state1 = state1_next
+            decoder_out, state0_next, state1_next = model.run_decoder(
+                ans[-1], state0, state1
+            )
+
+    ans = ans[1:]  # remove the first blank
+    print(ans)
+    tokens = [id2token[i] for i in ans]
+    underline = "▁"
+    #  underline = b"\xe2\x96\x81".decode()
+    text = "".join(tokens).replace(underline, " ").strip()
+    print("./example.wav")
+    print(text)
+
+
+if __name__ == "__main__":
+    main()

test_wavs/example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8aaaa18a5098d7c6de0595ae7ac1e64cacd0d4022af3595213bdaf23be77e69
+size 361324

test_wavs/long_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1868ece0195dfa9fc2394be24865d1133c8452a8292a397db45ba8c3ed9e01e3
+size 2280044

tokens.txt

@@ -0,0 +1,513 @@
+<unk> 0
+▁в 1
+▁п 2
+то 3
+▁с 4
+▁н 5
+▁к 6
+▁д 7
+▁о 8
+ст 9
+но 10
+▁и 11
+▁т 12
+▁м 13
+ра 14
+ен 15
+ро 16
+▁по 17
+▁на 18
+ет 19
+ли 20
+ак 21
+ер 22
+го 23
+ть 24
+▁б 25
+▁ч 26
+ль 27
+на 28
+ем 29
+▁э 30
+ри 31
+от 32
+ва 33
+ни 34
+▁з 35
+ла 36
+▁у 37
+ко 38
+ка 39
+ло 40
+▁не 41
+ре 42
+▁то 43
+ся 44
+во 45
+ны 46
+▁е 47
+▁это 48
+▁ко 49
+▁что 50
+▁а 51
+ле 52
+да 53
+ти 54
+де 55
+▁вот 56
+ки 57
+ди 58
+ам 59
+▁я 60
+те 61
+▁про 62
+▁как 63
+▁так 64
+▁да 65
+ру 66
+не 67
+ви 68
+ени 69
+▁мо 70
+же 71
+та 72
+сть 73
+▁ра 74
+чи 75
+▁во 76
+▁за 77
+▁со 78
+▁бы 79
+лю 80
+ма 81
+се 82
+ча 83
+ля 84
+▁до 85
+хо 86
+▁г 87
+ци 88
+▁ну 89
+ста 90
+▁ф 91
+▁при 92
+сь 93
+▁вы 94
+▁го 95
+▁се 96
+че 97
+▁но 98
+ми 99
+▁об 100
+▁от 101
+ве 102
+▁де 103
+за 104
+до 105
+▁все 106
+жи 107
+ры 108
+си 109
+ше 110
+сто 111
+▁те 112
+▁ка 113
+сти 114
+▁мы 115
+му 116
+ду 117
+ере 118
+лу 119
+по 120
+ще 121
+ку 122
+▁есть 123
+ой 124
+ень 125
+ну 126
+ца 127
+льно 128
+ско 129
+зо 130
+ски 131
+▁бу 132
+ши 133
+▁ма 134
+ется 135
+ня 136
+мо 137
+▁кото 138
+со 139
+▁ре 140
+ги 141
+▁бо 142
+▁он 143
+▁ни 144
+жа 145
+▁х 146
+жно 147
+га 148
+па 149
+бо 150
+ту 151
+ный 152
+енно 153
+▁из 154
+вы 155
+пи 156
+▁раз 157
+дца 158
+ные 159
+ют 160
+▁ли 161
+▁там 162
+ств 163
+вер 164
+ке 165
+▁ва 166
+▁зна 167
+мот 168
+са 169
+▁хо 170
+дет 171
+гда 172
+ять 173
+▁пре 174
+ме 175
+▁мне 176
+ери 177
+▁ви 178
+ты 179
+ения 180
+▁ро 181
+ря 182
+ение 183
+чно 184
+зи 185
+▁оч 186
+▁си 187
+вать 188
+▁которы 189
+▁ми 190
+▁ты 191
+▁фи 192
+ша 193
+каза 194
+▁под 195
+▁сам 196
+лько 197
+клю 198
+▁ме 199
+щи 200
+бы 201
+▁ве 202
+би 203
+▁очень 204
+▁пос 205
+тому 206
+вет 207
+дцать 208
+зы 209
+фи 210
+ных 211
+сли 212
+▁же 213
+бя 214
+ная 215
+▁пере 216
+▁сто 217
+▁ст 218
+▁два 219
+▁гово 220
+▁они 221
+▁для 222
+▁ин 223
+ете 224
+▁лю 225
+▁па 226
+нима 227
+пе 228
+▁сво 229
+ба 230
+▁три 231
+шь 232
+▁мен 233
+ут 234
+▁ку 235
+ска 236
+ной 237
+▁нас 238
+▁бе 239
+емь 240
+ного 241
+▁если 242
+бе 243
+це 244
+▁пер 245
+▁че 246
+▁вклю 247
+▁ста 248
+ство 249
+ему 250
+мен 251
+▁са 252
+▁сей 253
+▁будет 254
+▁ки 255
+час 256
+пу 257
+▁еще 258
+▁смот 259
+▁сейчас 260
+▁ча 261
+ды 262
+▁его 263
+дин 264
+рем 265
+▁эти 266
+▁ру 267
+▁уже 268
+▁она 269
+▁кон 270
+про 271
+гра 272
+сси 273
+▁дру 274
+▁меня 275
+ции 276
+ходи 277
+▁мож 278
+рет 279
+лы 280
+ное 281
+▁или 282
+▁пра 283
+ят 284
+▁жи 285
+▁таки 286
+ние 287
+бот 288
+ться 289
+▁включи 290
+ров 291
+▁су 292
+тельно 293
+▁ба 294
+чу 295
+об 296
+▁ш 297
+▁просто 298
+су 299
+мер 300
+ей 301
+▁филь 302
+▁когда 303
+▁сказа 304
+▁зак 305
+мы 306
+ая 307
+ран 308
+▁ше 309
+тель 310
+▁потому 311
+чески 312
+▁боль 313
+он 314
+▁вам 315
+▁вос 316
+дь 317
+▁од 318
+ую 319
+▁рас 320
+▁дол 321
+▁чтобы 322
+▁чет 323
+▁пока 324
+▁дев 325
+лся 326
+▁было 327
+деся 328
+нов 329
+▁пять 330
+став 331
+▁пред 332
+▁каки 333
+ое 334
+лен 335
+▁можно 336
+▁зде 337
+▁один 338
+▁здесь 339
+тся 340
+▁говори 341
+йте 342
+▁которые 343
+ния 344
+том 345
+гу 346
+▁воз 347
+▁ис 348
+▁тебя 349
+▁най 350
+▁сери 351
+тов 352
+▁ле 353
+▁ду 354
+ор 355
+▁нет 356
+зыва 357
+дела 358
+зу 359
+тив 360
+десят 361
+ха 362
+▁му 363
+ской 364
+▁работ 365
+▁дела 366
+лове 367
+ция 368
+лет 369
+▁может 370
+▁ди 371
+ным 372
+ственно 373
+бер 374
+бу 375
+▁врем 376
+▁им 377
+▁ско 378
+▁оп 379
+▁где 380
+▁сезо 381
+▁только 382
+год 383
+▁пи 384
+▁теле 385
+жен 386
+▁четы 387
+▁челове 388
+▁тоже 389
+рой 390
+лее 391
+▁ц 392
+▁та 393
+▁после 394
+▁дж 395
+▁росси 396
+▁быть 397
+▁сдела 398
+ром 399
+сы 400
+▁фильм 401
+▁хот 402
+лись 403
+▁сло 404
+ства 405
+���такой 406
+▁этот 407
+▁ю 408
+рав 409
+стви 410
+ал 411
+лось 412
+▁смотре 413
+ков 414
+▁семь 415
+▁коне 416
+▁этом 417
+▁ту 418
+▁полу 419
+▁кра 420
+▁тем 421
+при 422
+ную 423
+нал 424
+▁обра 425
+ом 426
+▁том 427
+шке 428
+▁был 429
+сно 430
+▁их 431
+реть 432
+кт 433
+годня 434
+▁четыре 435
+чит 436
+▁этого 437
+▁ска 438
+▁чем 439
+▁сегодня 440
+▁хоро 441
+▁конечно 442
+▁кар 443
+▁тв 444
+▁инт 445
+вой 446
+сте 447
+лей 448
+▁ти 449
+▁даже 450
+▁надо 451
+сот 452
+▁смотрешке 453
+▁пу 454
+ном 455
+▁вс 456
+рова 457
+▁какие 458
+лем 459
+ент 460
+лют 461
+вый 462
+▁сбер 463
+▁сказать 464
+ский 465
+она 466
+▁нам 467
+ского 468
+▁того 469
+нь 470
+▁ее 471
+▁ха 472
+▁какой 473
+▁стра 474
+▁каж 475
+▁ 476
+о 477
+е 478
+а 479
+т 480
+и 481
+н 482
+с 483
+р 484
+в 485
+л 486
+к 487
+м 488
+д 489
+у 490
+п 491
+ь 492
+я 493
+ы 494
+ч 495
+б 496
+з 497
+г 498
+й 499
+ж 500
+х 501
+ю 502
+ш 503
+э 504
+ц 505
+ф 506
+щ 507
+n 508
+ъ 509
+a 510
+ё 511
+<blk> 512