Training in progress, step 10

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

.ipynb_checkpoints/finetune-checkpoint.py

@@ -0,0 +1,267 @@
+import json
+import random
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+import torch
+import torchaudio
+import transformers
+import datasets
+from datasets import ClassLabel, load_dataset, load_metric
+from transformers import (Trainer, TrainingArguments, Wav2Vec2CTCTokenizer,
+                          Wav2Vec2FeatureExtractor, Wav2Vec2ForCTC,
+                          Wav2Vec2Processor)
+
+import argparse
+parser = argparse.ArgumentParser() 
+parser.add_argument('--model', type=str, default="facebook/wav2vec2-xls-r-300m")
+parser.add_argument('--unfreeze', action='store_true')
+parser.add_argument('--lr', type=float, default=3e-4)
+parser.add_argument('--warmup', type=float, default=500)
+args = parser.parse_args()
+
+
+print(f"args: {args}")
+
+common_voice_train = datasets.load_dataset("mozilla-foundation/common_voice_8_0", "zh-HK", split="train+validation", use_auth_token=True)
+common_voice_test = datasets.load_dataset("mozilla-foundation/common_voice_8_0", "zh-HK", split="test[:10%]", use_auth_token=True)
+
+# common_voice_train = datasets.load_dataset("common_voice", "zh-HK", split="train+validation", use_auth_token=True)
+# common_voice_test = datasets.load_dataset("common_voice", "zh-HK", split="test[:10%]", use_auth_token=True)
+
+unused_cols = ["accent", "age", "client_id", "down_votes", "gender", "locale", "segment", "up_votes"]
+common_voice_train = common_voice_train.remove_columns(unused_cols)
+common_voice_test = common_voice_test.remove_columns(unused_cols)
+
+chars_to_ignore_regex = '[\丶\,\?\.\!\-\;\:"\“\%\‘\”\�\．\⋯\！\－\：\–\。\》\,\）\,\？\；\～\~\…\︰\，\（\」\‧\《\﹔\、\—\／\,\「\﹖\·\']'
+
+import string
+def remove_special_characters(batch):
+    sen = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
+    # convert 'D' and 'd' to '啲' if there a 'D' in sentence
+    # hacky stuff, wont work on 'D', 'd' co-occure with normal english words
+    # wont work on multiple 'D'
+    if "d" in sen:
+        if len([c for c in sen if c in string.ascii_lowercase]) == 1:
+            sen = sen.replace("d", "啲")
+    batch["sentence"] = sen
+    return batch
+
+common_voice_train = common_voice_train.map(remove_special_characters)
+common_voice_test = common_voice_test.map(remove_special_characters)
+
+def extract_all_chars(batch):
+    all_text = " ".join(batch["sentence"])
+    vocab = list(set(all_text))
+    return {"vocab": [vocab], "all_text": [all_text]}
+
+vocab_train = common_voice_train.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=True, remove_columns=common_voice_train.column_names,)
+vocab_test = common_voice_test.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=True, remove_columns=common_voice_test.column_names,)
+vocab_list = list(set(vocab_train["vocab"][0]) | set(vocab_test["vocab"][0]))
+vocab_list = [char for char in vocab_list if not char.isascii()] # remove english char from vocab_list, so tokenizer will replace english with [UNK]
+vocab_list.append(" ") # previous will remove " " from vocab_list
+
+vocab_dict = {v: k for k, v in enumerate(vocab_list)}
+vocab_dict["|"] = vocab_dict[" "]
+del vocab_dict[" "]
+
+vocab_dict["[UNK]"] = len(vocab_dict)
+vocab_dict["[PAD]"] = len(vocab_dict)
+
+with open("vocab.json", "w") as vocab_file:
+    json.dump(vocab_dict, vocab_file)
+
+tokenizer = Wav2Vec2CTCTokenizer("./vocab.json", unk_token="[UNK]", pad_token="[PAD]", word_delimiter_token="|")
+
+feature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True, return_attention_mask=True,)
+
+processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)
+processor.save_pretrained("./finetuned-wav2vec2-xls-r-300m-cantonese")
+
+# resamplers = {
+#     48000: torchaudio.transforms.Resample(48000, 16000),
+#     44100: torchaudio.transforms.Resample(44100, 16000),
+# }
+
+# def load_and_resample(batch):
+#     speech_array, sampling_rate = torchaudio.load(batch["path"])
+#     batch["array"] = resamplers[sampling_rate](speech_array).squeeze().numpy()
+#     batch["sampling_rate"] = 16_000
+#     batch["target_text"] = batch["sentence"]
+#     return batch
+
+# common_voice_train = common_voice_train.map(load_and_resample, remove_columns=common_voice_train.column_names,)
+# common_voice_test = common_voice_test.map(load_and_resample, remove_columns=common_voice_test.column_names,)
+
+
+common_voice_train = common_voice_train.cast_column('audio', datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate))
+common_voice_test = common_voice_test.cast_column('audio', datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate))
+
+
+def prepare_dataset(batch):
+    batch["input_values"] = processor(batch["array"], sampling_rate=batch["sampling_rate"][0]).input_values
+    with processor.as_target_processor():
+        batch["labels"] = processor(batch["target_text"]).input_ids
+    return batch
+
+print(common_voice_train[0]['audio'])
+
+common_voice_train = common_voice_train.map(prepare_dataset, remove_columns=common_voice_train.column_names, batched=True,)
+common_voice_test = common_voice_test.map(prepare_dataset, remove_columns=common_voice_test.column_names,  batched=True,)
+
+
+@dataclass
+class DataCollatorCTCWithPadding:
+    """
+    Data collator that will dynamically pad the inputs received.
+    Args:
+        processor (:class:`~transformers.Wav2Vec2Processor`)
+            The processor used for proccessing the data.
+        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
+            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
+            among:
+            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+              sequence if provided).
+            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
+              maximum acceptable input length for the model if that argument is not provided.
+            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
+              different lengths).
+        max_length (:obj:`int`, `optional`):
+            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).
+        max_length_labels (:obj:`int`, `optional`):
+            Maximum length of the ``labels`` returned list and optionally padding length (see above).
+        pad_to_multiple_of (:obj:`int`, `optional`):
+            If set will pad the sequence to a multiple of the provided value.
+            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
+            7.5 (Volta).
+    """
+
+    processor: Wav2Vec2Processor
+    padding: Union[bool, str] = True
+    max_length: Optional[int] = None
+    max_length_labels: Optional[int] = None
+    pad_to_multiple_of: Optional[int] = None
+    pad_to_multiple_of_labels: Optional[int] = None
+
+    def __call__(
+        self, features: List[Dict[str, Union[List[int], torch.Tensor]]]
+    ) -> Dict[str, torch.Tensor]:
+        # split inputs and labels since they have to be of different lenghts and need
+        # different padding methods
+        input_features = [
+            {"input_values": feature["input_values"]} for feature in features
+        ]
+        label_features = [{"input_ids": feature["labels"]} for feature in features]
+
+        batch = self.processor.pad(
+            input_features,
+            padding=self.padding,
+            max_length=self.max_length,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors="pt",
+        )
+        with self.processor.as_target_processor():
+            labels_batch = self.processor.pad(
+                label_features,
+                padding=self.padding,
+                max_length=self.max_length_labels,
+                pad_to_multiple_of=self.pad_to_multiple_of_labels,
+                return_tensors="pt",
+            )
+
+        # replace padding with -100 to ignore loss correctly
+        labels = labels_batch["input_ids"].masked_fill(
+            labels_batch.attention_mask.ne(1), -100
+        )
+
+        batch["labels"] = labels
+
+        return batch
+
+
+data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)
+# cer_metric = load_metric("./cer")
+
+# def compute_metrics(pred):
+#     pred_logits = pred.predictions
+#     pred_ids = np.argmax(pred_logits, axis=-1)
+
+#     pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id
+
+#     pred_str = processor.batch_decode(pred_ids)
+#     # we do not want to group tokens when computing the metrics
+#     label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
+
+#     cer = cer_metric.compute(predictions=pred_str, references=label_str)
+
+#     return {"cer": cer}
+
+def compute_metrics(pred):
+        pred_logits = pred.predictions
+        pred_ids = np.argmax(pred_logits, axis=-1)
+
+        pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id
+
+        pred_str = tokenizer.batch_decode(pred_ids)
+        # we do not want to group tokens when computing the metrics
+        label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)
+
+        metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()}
+
+        return metrics
+
+model = Wav2Vec2ForCTC.from_pretrained(
+    args.model,
+    attention_dropout=0.1,
+    hidden_dropout=0.1,
+    feat_proj_dropout=0.0,
+    mask_time_prob=0.05,
+    layerdrop=0.1,
+    gradient_checkpointing=True,
+    ctc_loss_reduction="mean",
+    pad_token_id=processor.tokenizer.pad_token_id,
+    vocab_size=len(processor.tokenizer),
+)
+
+if not args.unfreeze:
+    model.freeze_feature_extractor()
+
+training_args = TrainingArguments(
+    output_dir="./finetuned-wav2vec2-xls-r-300m-cantonese/wav2vec2-xls-r-300m-cantonese",
+    group_by_length=True,
+    per_device_train_batch_size=8,
+    gradient_accumulation_steps=2,
+    #evaluation_strategy="no",
+    evaluation_strategy="steps",
+    #evaluation_strategy="epoch",
+    eval_steps=400,
+    #eval_accumulation_steps=60,
+    num_train_epochs=1,
+    fp16=True,
+    fp16_backend="amp",
+    logging_strategy="steps",
+    logging_steps=400,
+    #logging_strategy="epoch",
+    learning_rate=args.lr,
+    warmup_steps=100,
+    save_steps=2376, # every 3 epoch with batch_size 8
+    #save_strategy="epoch",
+    save_total_limit=3,
+    ###################
+    # fp16_full_eval=True,
+    dataloader_num_workers=20,
+)
+
+trainer = Trainer(
+    model=model,
+    data_collator=data_collator,
+    args=training_args,
+    compute_metrics=compute_metrics,
+    train_dataset=common_voice_train,
+    eval_dataset=common_voice_test,
+    tokenizer=processor.feature_extractor,
+)
+trainer.train()

.ipynb_checkpoints/run-checkpoint.sh

@@ -0,0 +1,29 @@
+python run_speech_recognition_ctc.py \
+	--dataset_name="mozilla-foundation/common_voice_8_0" \
+	--model_name_or_path="facebook/wav2vec2-xls-r-300m" \
+	--dataset_config_name="zh-HK" \
+	--output_dir="./" \
+    --cache_dir="../container_0" \
+	--overwrite_output_dir \
+	--num_train_epochs="1" \
+	--per_device_train_batch_size="8" \
+    --per_device_eval_batch_size="1" \
+	--gradient_accumulation_steps="2" \
+	--learning_rate="3e-4" \
+	--warmup_steps="500" \
+	--evaluation_strategy="steps" \
+	--text_column_name="sentence" \
+	--length_column_name="input_length" \
+	--save_steps="10" \
+	--eval_steps="10" \
+	--layerdrop="0.0" \
+	--save_total_limit="3" \
+	--freeze_feature_encoder \
+	--gradient_checkpointing \
+	--fp16 \
+	--group_by_length \
+    --use_auth_token \
+	--push_to_hub \
+	--do_train \
+    --do_eval \
+	--max_duration_in_seconds="3"

.ipynb_checkpoints/run_speech_recognition_ctc-checkpoint.py

@@ -0,0 +1,780 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
+
+import functools
+import json
+import logging
+import os
+import re
+import sys
+import warnings
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
+
+import datasets
+import numpy as np
+import torch
+from datasets import DatasetDict, load_dataset, load_metric
+
+import transformers
+from transformers import (
+    AutoConfig,
+    AutoFeatureExtractor,
+    AutoModelForCTC,
+    AutoProcessor,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    Wav2Vec2Processor,
+    set_seed,
+)
+from transformers.trainer_utils import get_last_checkpoint, is_main_process
+from transformers.utils import check_min_version
+from transformers.utils.versions import require_version
+
+import string
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.17.0.dev0")
+
+require_version("datasets>=1.13.3", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt")
+
+
+logger = logging.getLogger(__name__)
+
+
+def list_field(default=None, metadata=None):
+    return field(default_factory=lambda: default, metadata=metadata)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={"help": "Path to pretrained tokenizer or tokenizer identifier from huggingface.co/models"},
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    freeze_feature_encoder: bool = field(
+        default=True, metadata={"help": "Whether to freeze the feature encoder layers of the model."}
+    )
+    attention_dropout: float = field(
+        default=0.0, metadata={"help": "The dropout ratio for the attention probabilities."}
+    )
+    activation_dropout: float = field(
+        default=0.0, metadata={"help": "The dropout ratio for activations inside the fully connected layer."}
+    )
+    feat_proj_dropout: float = field(default=0.0, metadata={"help": "The dropout ratio for the projected features."})
+    hidden_dropout: float = field(
+        default=0.0,
+        metadata={
+            "help": "The dropout probability for all fully connected layers in the embeddings, encoder, and pooler."
+        },
+    )
+    final_dropout: float = field(
+        default=0.0,
+        metadata={"help": "The dropout probability for the final projection layer."},
+    )
+    mask_time_prob: float = field(
+        default=0.05,
+        metadata={
+            "help": "Probability of each feature vector along the time axis to be chosen as the start of the vector"
+            "span to be masked. Approximately ``mask_time_prob * sequence_length // mask_time_length`` feature"
+            "vectors will be masked along the time axis."
+        },
+    )
+    mask_time_length: int = field(
+        default=10,
+        metadata={"help": "Length of vector span to mask along the time axis."},
+    )
+    mask_feature_prob: float = field(
+        default=0.0,
+        metadata={
+            "help": "Probability of each feature vector along the feature axis to be chosen as the start of the vector"
+            "span to be masked. Approximately ``mask_feature_prob * sequence_length // mask_feature_length`` feature bins will be masked along the time axis."
+        },
+    )
+    mask_feature_length: int = field(
+        default=10,
+        metadata={"help": "Length of vector span to mask along the feature axis."},
+    )
+    layerdrop: float = field(default=0.0, metadata={"help": "The LayerDrop probability."})
+    ctc_loss_reduction: Optional[str] = field(
+        default="mean", metadata={"help": "The way the ctc loss should be reduced. Should be one of 'mean' or 'sum'."}
+    )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+
+    Using `HfArgumentParser` we can turn this class
+    into argparse arguments to be able to specify them on
+    the command line.
+    """
+
+    dataset_name: str = field(
+        metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: str = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_split_name: str = field(
+        default="train+validation",
+        metadata={
+            "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train+validation'"
+        },
+    )
+    eval_split_name: str = field(
+        default="test",
+        metadata={
+            "help": "The name of the evaluation data set split to use (via the datasets library). Defaults to 'test'"
+        },
+    )
+    audio_column_name: str = field(
+        default="audio",
+        metadata={"help": "The name of the dataset column containing the audio data. Defaults to 'audio'"},
+    )
+    text_column_name: str = field(
+        default="text",
+        metadata={"help": "The name of the dataset column containing the text data. Defaults to 'text'"},
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
+            "value if set."
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "value if set."
+        },
+    )
+    chars_to_ignore: List[str] = list_field(
+        default=[",", "?", "¿", ".", "!", "¡", ";", "；", ":", '""', "%", '"', "�", "ʿ", "·", "჻", "~", "՞",
+                 "؟", "،", "।", "॥", "«", "»", "„", "“", "”", "「", "」", "‘", "’", "《", "》", "(", ")", "[", "]",
+                 "{", "}", "=", "`", "_", "+", "<", ">", "…", "–", "°", "´", "ʾ", "‹", "›", "©", "®", "—", "→", "。",
+                 "、", "﹂", "﹁", "‧", "～", "﹏", "，", "｛", "｝", "（", "）", "［", "］", "【", "】", "‥", "〽",
+                 "『", "』", "〝", "〟", "⟨", "⟩", "〜", "：", "！", "？", "♪", "؛", "/", "\\", "º", "−", "^", "ʻ", "ˆ"],
+        metadata={"help": "A list of characters to remove from the transcripts."},
+    )
+    eval_metrics: List[str] = list_field(
+        default=["wer"],
+        metadata={"help": "A list of metrics the model should be evaluated on. E.g. `'wer cer'`"},
+    )
+    max_duration_in_seconds: float = field(
+        default=20.0,
+        metadata={
+            "help": "Filter audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`"
+        },
+    )
+    min_duration_in_seconds: float = field(
+        default=0.0, metadata={"help": "Filter audio files that are shorter than `min_duration_in_seconds` seconds"}
+    )
+    preprocessing_only: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether to only do data preprocessing and skip training. "
+            "This is especially useful when data preprocessing errors out in distributed training due to timeout. "
+            "In this case, one should run the preprocessing in a non-distributed setup with `preprocessing_only=True` "
+            "so that the cached datasets can consequently be loaded in distributed training"
+        },
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": "If :obj:`True`, will use the token generated when running"
+            ":obj:`transformers-cli login` as HTTP bearer authorization for remote files."
+        },
+    )
+    unk_token: str = field(
+        default="[UNK]",
+        metadata={"help": "The unk token for the tokenizer"},
+    )
+    pad_token: str = field(
+        default="[PAD]",
+        metadata={"help": "The padding token for the tokenizer"},
+    )
+    word_delimiter_token: str = field(
+        default="|",
+        metadata={"help": "The word delimiter token for the tokenizer"},
+    )
+    phoneme_language: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The target language that should be used be"
+            " passed to the tokenizer for tokenization. Note that"
+            " this is only relevant if the model classifies the"
+            " input audio to a sequence of phoneme sequences."
+        },
+    )
+
+
+@dataclass
+class DataCollatorCTCWithPadding:
+    """
+    Data collator that will dynamically pad the inputs received.
+    Args:
+        processor (:class:`~transformers.AutoProcessor`)
+            The processor used for proccessing the data.
+        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
+            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
+            among:
+            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+              sequence if provided).
+            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
+              maximum acceptable input length for the model if that argument is not provided.
+            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
+              different lengths).
+        max_length (:obj:`int`, `optional`):
+            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).
+        max_length_labels (:obj:`int`, `optional`):
+            Maximum length of the ``labels`` returned list and optionally padding length (see above).
+        pad_to_multiple_of (:obj:`int`, `optional`):
+            If set will pad the sequence to a multiple of the provided value.
+            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
+            7.5 (Volta).
+    """
+
+    processor: AutoProcessor
+    padding: Union[bool, str] = "longest"
+    pad_to_multiple_of: Optional[int] = None
+    pad_to_multiple_of_labels: Optional[int] = None
+
+    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
+        # split inputs and labels since they have to be of different lenghts and need
+        # different padding methods
+        input_features = [{"input_values": feature["input_values"]} for feature in features]
+        label_features = [{"input_ids": feature["labels"]} for feature in features]
+
+        batch = self.processor.pad(
+            input_features,
+            padding=self.padding,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors="pt",
+        )
+
+        with self.processor.as_target_processor():
+            labels_batch = self.processor.pad(
+                label_features,
+                padding=self.padding,
+                pad_to_multiple_of=self.pad_to_multiple_of_labels,
+                return_tensors="pt",
+            )
+
+        # replace padding with -100 to ignore loss correctly
+        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)
+
+        batch["labels"] = labels
+
+        return batch
+
+
+def create_vocabulary_from_data(
+    datasets: DatasetDict,
+    word_delimiter_token: Optional[str] = None,
+    unk_token: Optional[str] = None,
+    pad_token: Optional[str] = None,
+):
+    # Given training and test labels create vocabulary
+    def extract_all_chars(batch):
+        all_text = " ".join(batch["target_text"])
+        vocab = list(set(all_text))
+        return {"vocab": [vocab], "all_text": [all_text]}
+
+    vocabs = datasets.map(
+        extract_all_chars,
+        batched=True,
+        batch_size=-1,
+        keep_in_memory=True,
+        remove_columns=datasets["train"].column_names,
+    )
+
+    # take union of all unique characters in each dataset
+    vocab_set = functools.reduce(
+        lambda vocab_1, vocab_2: set(vocab_1["vocab"][0]) | set(vocab_2["vocab"][0]), vocabs.values()
+    )
+
+    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocab_set)))}
+
+    # replace white space with delimiter token
+    if word_delimiter_token is not None:
+        vocab_dict[word_delimiter_token] = vocab_dict[" "]
+        del vocab_dict[" "]
+
+    # add unk and pad token
+    if unk_token is not None:
+        vocab_dict[unk_token] = len(vocab_dict)
+
+    if pad_token is not None:
+        vocab_dict[pad_token] = len(vocab_dict)
+
+    return vocab_dict
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # 1. First, let's load the dataset
+    raw_datasets = DatasetDict()
+
+    if training_args.do_train:
+        raw_datasets["train"]  = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            split=data_args.train_split_name,
+            use_auth_token=data_args.use_auth_token,
+        )
+
+        if data_args.audio_column_name not in raw_datasets["train"].column_names:
+            raise ValueError(
+                f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'. "
+                "Make sure to set `--audio_column_name` to the correct audio column - one of "
+                f"{', '.join(raw_datasets['train'].column_names)}."
+            )
+
+        if data_args.text_column_name not in raw_datasets["train"].column_names:
+            raise ValueError(
+                f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. "
+                "Make sure to set `--text_column_name` to the correct text column - one of "
+                f"{', '.join(raw_datasets['train'].column_names)}."
+            )
+
+        if data_args.max_train_samples is not None:
+            raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples))
+
+    if training_args.do_eval:
+        raw_datasets["eval"] = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            split=data_args.eval_split_name,
+            use_auth_token=data_args.use_auth_token,
+        )
+
+        if data_args.max_eval_samples is not None:
+            raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples))
+
+    # 2. We remove some special characters from the datasets
+    # that make training complicated and do not help in transcribing the speech
+    # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
+    # that could be easily picked up by the model
+    chars_to_ignore_regex = (
+        f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
+    )
+    text_column_name = data_args.text_column_name
+
+    def remove_special_characters(batch):
+        if chars_to_ignore_regex is not None:
+            sen = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " "
+            
+            # convert 'D' and 'd' to '啲' if there a 'D' in sentence
+            # hacky stuff, wont work on 'D', 'd' co-occure with normal english words
+            # wont work on multiple 'D'
+            if "d" in sen:
+                if len([c for c in sen if c in string.ascii_lowercase]) == 1:
+                    sen = sen.replace("d", "啲")
+            batch["target_text"] = sen    
+        else:
+            batch["target_text"] = batch[text_column_name].lower() + " "
+        return batch
+
+    with training_args.main_process_first(desc="dataset map special characters removal"):
+        raw_datasets = raw_datasets.map(
+            remove_special_characters,
+            remove_columns=[text_column_name],
+            desc="remove special characters from datasets",
+        )
+
+    # save special tokens for tokenizer
+    word_delimiter_token = data_args.word_delimiter_token
+    unk_token = data_args.unk_token
+    pad_token = data_args.pad_token
+
+    # 3. Next, let's load the config as we might need it to create
+    # the tokenizer
+    # load config
+    config = AutoConfig.from_pretrained(
+        model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
+    )
+
+    # 4. Next, if no tokenizer file is defined,
+    # we create the vocabulary of the model by extracting all unique characters from
+    # the training and evaluation datasets
+    # We need to make sure that only first rank saves vocabulary
+    # make sure all processes wait until vocab is created
+    tokenizer_name_or_path = model_args.tokenizer_name_or_path
+    tokenizer_kwargs = {}
+    if tokenizer_name_or_path is None:
+        # save vocab in training output dir
+        tokenizer_name_or_path = training_args.output_dir
+
+        vocab_file = os.path.join(tokenizer_name_or_path, "vocab.json")
+
+        with training_args.main_process_first():
+            if training_args.overwrite_output_dir and os.path.isfile(vocab_file):
+                os.remove(vocab_file)
+
+        with training_args.main_process_first(desc="dataset map vocabulary creation"):
+            if not os.path.isfile(vocab_file):
+                os.makedirs(tokenizer_name_or_path, exist_ok=True)
+                vocab_dict = create_vocabulary_from_data(
+                    raw_datasets,
+                    word_delimiter_token=word_delimiter_token,
+                    unk_token=unk_token,
+                    pad_token=pad_token,
+                )
+
+                # save vocab dict to be loaded into tokenizer
+                with open(vocab_file, "w") as file:
+                    json.dump(vocab_dict, file)
+
+        # if tokenizer has just been created
+        # it is defined by `tokenizer_class` if present in config else by `model_type`
+        tokenizer_kwargs = {
+            "config": config if config.tokenizer_class is not None else None,
+            "tokenizer_type": config.model_type if config.tokenizer_class is None else None,
+            "unk_token": unk_token,
+            "pad_token": pad_token,
+            "word_delimiter_token": word_delimiter_token,
+        }
+
+    # 5. Now we can instantiate the feature extractor, tokenizer and model
+    # Note for distributed training, the .from_pretrained methods guarantee that only
+    # one local process can concurrently download model & vocab.
+
+    # load feature_extractor and tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_name_or_path,
+        use_auth_token=data_args.use_auth_token,
+        **tokenizer_kwargs,
+    )
+    feature_extractor = AutoFeatureExtractor.from_pretrained(
+        model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
+    )
+
+    # adapt config
+    config.update(
+        {
+            "feat_proj_dropout": model_args.feat_proj_dropout,
+            "attention_dropout": model_args.attention_dropout,
+            "hidden_dropout": model_args.hidden_dropout,
+            "final_dropout": model_args.final_dropout,
+            "mask_time_prob": model_args.mask_time_prob,
+            "mask_time_length": model_args.mask_time_length,
+            "mask_feature_prob": model_args.mask_feature_prob,
+            "mask_feature_length": model_args.mask_feature_length,
+            "gradient_checkpointing": training_args.gradient_checkpointing,
+            "layerdrop": model_args.layerdrop,
+            "ctc_loss_reduction": model_args.ctc_loss_reduction,
+            "pad_token_id": tokenizer.pad_token_id,
+            "vocab_size": len(tokenizer),
+            "activation_dropout": model_args.activation_dropout,
+        }
+    )
+
+    # create model
+    model = AutoModelForCTC.from_pretrained(
+        model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        config=config,
+        use_auth_token=data_args.use_auth_token,
+    )
+
+    # freeze encoder
+    if model_args.freeze_feature_encoder:
+        model.freeze_feature_encoder()
+
+    # 6. Now we preprocess the datasets including remove long audio sample, loading the audio, resampling and normalization
+    # Thankfully, `datasets` takes care of automatically loading and resampling the audio,
+    # so that we just need to set the correct target sampling rate and normalize the input
+    # via the `feature_extractor`
+
+    # make sure that dataset decodes audio with correct sampling rate
+    dataset_sampling_rate = next(iter(raw_datasets.values())).features[data_args.audio_column_name].sampling_rate
+    # print("data sample rate:", dataset_sampling_rate) # 48_000
+    # print("feature sample rate:", feature_extractor.sampling_rate) # 16_000
+
+    # # remove long common voice
+    # def remove_long_common_voicedata(dataset, max_seconds=6):
+    #     #convert pyarrow table to pandas
+    #     dftest= dataset.to_pandas()
+
+    #     #find out length of input_values
+    #     dftest['len']= dftest['target_text'].apply(len)
+
+    #     #for wav2vec training we already resampled to 16khz
+    #     #remove data that is longer than max_seconds (6 seconds ideal)
+    #     maxLength = max_seconds * 16000 
+    #     dftest= dftest[dftest['len']<maxLength]
+    #     dftest = dftest.drop('len', 1)
+
+    #     #convert back to pyarrow table to use in trainer
+    #     dataset= dataset.from_pandas(dftest)
+
+    #     #directly remove do not wait for gc
+    #     del dftest
+    #     return dataset
+
+    # raw_datasets['train'] = remove_long_common_voicedata(raw_datasets['train'], max_seconds=3)
+    # raw_datasets['eval'] = remove_long_common_voicedata(raw_datasets['eval'], max_seconds=3)
+
+
+    # casting column
+    if dataset_sampling_rate != feature_extractor.sampling_rate:
+        raw_datasets = raw_datasets.cast_column(
+            data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate)
+        )
+
+
+    # derive max & min input length for sample rate & max duration
+    max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate
+    min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate
+    audio_column_name = data_args.audio_column_name
+    num_workers = data_args.preprocessing_num_workers
+
+    # `phoneme_language` is only relevant if the model is fine-tuned on phoneme classification
+    phoneme_language = data_args.phoneme_language
+
+    # Preprocessing the datasets.
+    # We need to read the audio files as arrays and tokenize the targets.
+    def prepare_dataset(batch):
+        # load audio
+        sample = batch[audio_column_name]
+
+        inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"])
+        batch["input_values"] = inputs.input_values[0]
+        batch["input_length"] = len(batch["input_values"])
+
+        # encode targets
+        additional_kwargs = {}
+        if phoneme_language is not None:
+            additional_kwargs["phonemizer_lang"] = phoneme_language
+
+        batch["labels"] = tokenizer(batch["target_text"], **additional_kwargs).input_ids
+        return batch
+
+    with training_args.main_process_first(desc="dataset map preprocessing"):
+        vectorized_datasets = raw_datasets.map(
+            prepare_dataset,
+            remove_columns=next(iter(raw_datasets.values())).column_names,
+            num_proc=num_workers,
+            desc="preprocess datasets",
+        )
+
+        def is_audio_in_length_range(length):
+            return length > min_input_length and length < max_input_length
+
+        # filter data that is shorter than min_input_length
+        vectorized_datasets = vectorized_datasets.filter(
+            is_audio_in_length_range,
+            num_proc=num_workers,
+            input_columns=["input_length"],
+        )
+
+    # 7. Next, we can prepare the training.
+    # Let's use word error rate (WER) as our evaluation metric,
+    # instantiate a data collator and the trainer
+
+    # Define evaluation metrics during training, *i.e.* word error rate, character error rate
+    eval_metrics = {metric: load_metric(metric) for metric in data_args.eval_metrics}
+
+    # for large datasets it is advised to run the preprocessing on a
+    # single machine first with ``args.preprocessing_only`` since there will mostly likely
+    # be a timeout when running the script in distributed mode.
+    # In a second step ``args.preprocessing_only`` can then be set to `False` to load the
+    # cached dataset
+    if data_args.preprocessing_only:
+        logger.info(f"Data preprocessing finished. Files cached at {vectorized_datasets.cache_files}")
+        return
+
+    def compute_metrics(pred):
+        pred_logits = pred.predictions
+        pred_ids = np.argmax(pred_logits, axis=-1)
+
+        pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id
+
+        pred_str = tokenizer.batch_decode(pred_ids)
+        # we do not want to group tokens when computing the metrics
+        label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)
+
+        metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()}
+
+        return metrics
+
+    # Now save everything to be able to create a single processor later
+    if is_main_process(training_args.local_rank):
+        # save feature extractor, tokenizer and config
+        feature_extractor.save_pretrained(training_args.output_dir)
+        tokenizer.save_pretrained(training_args.output_dir)
+        config.save_pretrained(training_args.output_dir)
+
+    try:
+        processor = AutoProcessor.from_pretrained(training_args.output_dir)
+    except (OSError, KeyError):
+        warnings.warn(
+            "Loading a processor from a feature extractor config that does not"
+            " include a `processor_class` attribute is deprecated and will be removed in v5. Please add the following "
+            " attribute to your `preprocessor_config.json` file to suppress this warning: "
+            " `'processor_class': 'Wav2Vec2Processor'`",
+            FutureWarning,
+        )
+        processor = Wav2Vec2Processor.from_pretrained(training_args.output_dir)
+
+    # Instantiate custom data collator
+    data_collator = DataCollatorCTCWithPadding(processor=processor)
+
+    # Initialize Trainer
+    trainer = Trainer(
+        model=model,
+        data_collator=data_collator,
+        args=training_args,
+        compute_metrics=compute_metrics,
+        train_dataset=vectorized_datasets["train"] if training_args.do_train else None,
+        eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None,
+        tokenizer=feature_extractor,
+    )
+
+    # 8. Finally, we can start training
+
+    # Training
+    if training_args.do_train:
+
+        # use last checkpoint if exist
+        if last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        elif os.path.isdir(model_args.model_name_or_path):
+            checkpoint = model_args.model_name_or_path
+        else:
+            checkpoint = None
+
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()
+
+        metrics = train_result.metrics
+        max_train_samples = (
+            data_args.max_train_samples
+            if data_args.max_train_samples is not None
+            else len(vectorized_datasets["train"])
+        )
+        metrics["train_samples"] = min(max_train_samples, len(vectorized_datasets["train"]))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    results = {}
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+        metrics = trainer.evaluate()
+        max_eval_samples = (
+            data_args.max_eval_samples if data_args.max_eval_samples is not None else len(vectorized_datasets["eval"])
+        )
+        metrics["eval_samples"] = min(max_eval_samples, len(vectorized_datasets["eval"]))
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    # Write model card and (optionally) push to hub
+    config_name = data_args.dataset_config_name if data_args.dataset_config_name is not None else "na"
+    kwargs = {
+        "finetuned_from": model_args.model_name_or_path,
+        "tasks": "speech-recognition",
+        "tags": ["automatic-speech-recognition", data_args.dataset_name],
+        "dataset_args": f"Config: {config_name}, Training split: {data_args.train_split_name}, Eval split: {data_args.eval_split_name}",
+        "dataset": f"{data_args.dataset_name.upper()} - {config_name.upper()}",
+    }
+    if "common_voice" in data_args.dataset_name:
+        kwargs["language"] = config_name
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 3925, "</s>": 3926}

config.json

@@ -0,0 +1,107 @@
+{
+  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.0,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 3924,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 3927,
+  "xvector_output_dim": 512
+}

eval.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+import argparse
+import re
+from typing import Dict
+
+import torch
+from datasets import Audio, Dataset, load_dataset, load_metric
+
+from transformers import AutoFeatureExtractor, pipeline
+
+
+def log_results(result: Dataset, args: Dict[str, str]):
+    """DO NOT CHANGE. This function computes and logs the result metrics."""
+
+    log_outputs = args.log_outputs
+    dataset_id = "_".join(args.dataset.split("/") + [args.config, args.split])
+
+    # load metric
+    wer = load_metric("wer")
+    cer = load_metric("cer")
+
+    # compute metrics
+    wer_result = wer.compute(references=result["target"], predictions=result["prediction"])
+    cer_result = cer.compute(references=result["target"], predictions=result["prediction"])
+
+    # print & log results
+    result_str = f"WER: {wer_result}\n" f"CER: {cer_result}"
+    print(result_str)
+
+    with open(f"{dataset_id}_eval_results.txt", "w") as f:
+        f.write(result_str)
+
+    # log all results in text file. Possibly interesting for analysis
+    if log_outputs is not None:
+        pred_file = f"log_{dataset_id}_predictions.txt"
+        target_file = f"log_{dataset_id}_targets.txt"
+
+        with open(pred_file, "w") as p, open(target_file, "w") as t:
+
+            # mapping function to write output
+            def write_to_file(batch, i):
+                p.write(f"{i}" + "\n")
+                p.write(batch["prediction"] + "\n")
+                t.write(f"{i}" + "\n")
+                t.write(batch["target"] + "\n")
+
+            result.map(write_to_file, with_indices=True)
+
+
+def normalize_text(text: str) -> str:
+    """DO ADAPT FOR YOUR USE CASE. this function normalizes the target text."""
+
+    chars_to_ignore_regex = '[,?.!\-\;\:"“%‘”�—’…–]'  # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
+
+    text = re.sub(chars_to_ignore_regex, "", text.lower())
+
+    # In addition, we can normalize the target text, e.g. removing new lines characters etc...
+    # note that order is important here!
+    token_sequences_to_ignore = ["\n\n", "\n", "   ", "  "]
+
+    for t in token_sequences_to_ignore:
+        text = " ".join(text.split(t))
+
+    return text
+
+
+def main(args):
+    # load dataset
+    dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
+
+    # for testing: only process the first two examples as a test
+    # dataset = dataset.select(range(10))
+
+    # load processor
+    feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
+    sampling_rate = feature_extractor.sampling_rate
+
+    # resample audio
+    dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate))
+
+    # load eval pipeline
+    if args.device is None:
+        args.device = 0 if torch.cuda.is_available() else -1
+    asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
+
+    # map function to decode audio
+    def map_to_pred(batch):
+        prediction = asr(
+            batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s
+        )
+
+        batch["prediction"] = prediction["text"]
+        batch["target"] = normalize_text(batch["sentence"])
+        return batch
+
+    # run inference on all examples
+    result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
+
+    # compute and log_results
+    # do not change function below
+    log_results(result, args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers"
+    )
+    parser.add_argument(
+        "--dataset",
+        type=str,
+        required=True,
+        help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets",
+    )
+    parser.add_argument(
+        "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'`  for Common Voice"
+    )
+    parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`")
+    parser.add_argument(
+        "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
+    )
+    parser.add_argument(
+        "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to 1 second."
+    )
+    parser.add_argument(
+        "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis."
+    )
+    parser.add_argument(
+        "--device",
+        type=int,
+        default=None,
+        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
+    )
+    args = parser.parse_args()
+
+    main(args)

finetune.py

@@ -0,0 +1,267 @@
+import json
+import random
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+import torch
+import torchaudio
+import transformers
+import datasets
+from datasets import ClassLabel, load_dataset, load_metric
+from transformers import (Trainer, TrainingArguments, Wav2Vec2CTCTokenizer,
+                          Wav2Vec2FeatureExtractor, Wav2Vec2ForCTC,
+                          Wav2Vec2Processor)
+
+import argparse
+parser = argparse.ArgumentParser() 
+parser.add_argument('--model', type=str, default="facebook/wav2vec2-xls-r-300m")
+parser.add_argument('--unfreeze', action='store_true')
+parser.add_argument('--lr', type=float, default=3e-4)
+parser.add_argument('--warmup', type=float, default=500)
+args = parser.parse_args()
+
+
+print(f"args: {args}")
+
+common_voice_train = datasets.load_dataset("mozilla-foundation/common_voice_8_0", "zh-HK", split="train+validation", use_auth_token=True)
+common_voice_test = datasets.load_dataset("mozilla-foundation/common_voice_8_0", "zh-HK", split="test[:10%]", use_auth_token=True)
+
+# common_voice_train = datasets.load_dataset("common_voice", "zh-HK", split="train+validation", use_auth_token=True)
+# common_voice_test = datasets.load_dataset("common_voice", "zh-HK", split="test[:10%]", use_auth_token=True)
+
+unused_cols = ["accent", "age", "client_id", "down_votes", "gender", "locale", "segment", "up_votes"]
+common_voice_train = common_voice_train.remove_columns(unused_cols)
+common_voice_test = common_voice_test.remove_columns(unused_cols)
+
+chars_to_ignore_regex = '[\丶\,\?\.\!\-\;\:"\“\%\‘\”\�\．\⋯\！\－\：\–\。\》\,\）\,\？\；\～\~\…\︰\，\（\」\‧\《\﹔\、\—\／\,\「\﹖\·\']'
+
+import string
+def remove_special_characters(batch):
+    sen = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
+    # convert 'D' and 'd' to '啲' if there a 'D' in sentence
+    # hacky stuff, wont work on 'D', 'd' co-occure with normal english words
+    # wont work on multiple 'D'
+    if "d" in sen:
+        if len([c for c in sen if c in string.ascii_lowercase]) == 1:
+            sen = sen.replace("d", "啲")
+    batch["sentence"] = sen
+    return batch
+
+common_voice_train = common_voice_train.map(remove_special_characters)
+common_voice_test = common_voice_test.map(remove_special_characters)
+
+def extract_all_chars(batch):
+    all_text = " ".join(batch["sentence"])
+    vocab = list(set(all_text))
+    return {"vocab": [vocab], "all_text": [all_text]}
+
+vocab_train = common_voice_train.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=True, remove_columns=common_voice_train.column_names,)
+vocab_test = common_voice_test.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=True, remove_columns=common_voice_test.column_names,)
+vocab_list = list(set(vocab_train["vocab"][0]) | set(vocab_test["vocab"][0]))
+vocab_list = [char for char in vocab_list if not char.isascii()] # remove english char from vocab_list, so tokenizer will replace english with [UNK]
+vocab_list.append(" ") # previous will remove " " from vocab_list
+
+vocab_dict = {v: k for k, v in enumerate(vocab_list)}
+vocab_dict["|"] = vocab_dict[" "]
+del vocab_dict[" "]
+
+vocab_dict["[UNK]"] = len(vocab_dict)
+vocab_dict["[PAD]"] = len(vocab_dict)
+
+with open("vocab.json", "w") as vocab_file:
+    json.dump(vocab_dict, vocab_file)
+
+tokenizer = Wav2Vec2CTCTokenizer("./vocab.json", unk_token="[UNK]", pad_token="[PAD]", word_delimiter_token="|")
+
+feature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True, return_attention_mask=True,)
+
+processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)
+processor.save_pretrained("./finetuned-wav2vec2-xls-r-300m-cantonese")
+
+# resamplers = {
+#     48000: torchaudio.transforms.Resample(48000, 16000),
+#     44100: torchaudio.transforms.Resample(44100, 16000),
+# }
+
+# def load_and_resample(batch):
+#     speech_array, sampling_rate = torchaudio.load(batch["path"])
+#     batch["array"] = resamplers[sampling_rate](speech_array).squeeze().numpy()
+#     batch["sampling_rate"] = 16_000
+#     batch["target_text"] = batch["sentence"]
+#     return batch
+
+# common_voice_train = common_voice_train.map(load_and_resample, remove_columns=common_voice_train.column_names,)
+# common_voice_test = common_voice_test.map(load_and_resample, remove_columns=common_voice_test.column_names,)
+
+
+common_voice_train = common_voice_train.cast_column('audio', datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate))
+common_voice_test = common_voice_test.cast_column('audio', datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate))
+
+
+def prepare_dataset(batch):
+    batch["input_values"] = processor(batch["array"], sampling_rate=batch["sampling_rate"][0]).input_values
+    with processor.as_target_processor():
+        batch["labels"] = processor(batch["target_text"]).input_ids
+    return batch
+
+print(common_voice_train[0]['audio'])
+
+common_voice_train = common_voice_train.map(prepare_dataset, remove_columns=common_voice_train.column_names, batched=True,)
+common_voice_test = common_voice_test.map(prepare_dataset, remove_columns=common_voice_test.column_names,  batched=True,)
+
+
+@dataclass
+class DataCollatorCTCWithPadding:
+    """
+    Data collator that will dynamically pad the inputs received.
+    Args:
+        processor (:class:`~transformers.Wav2Vec2Processor`)
+            The processor used for proccessing the data.
+        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
+            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
+            among:
+            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+              sequence if provided).
+            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
+              maximum acceptable input length for the model if that argument is not provided.
+            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
+              different lengths).
+        max_length (:obj:`int`, `optional`):
+            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).
+        max_length_labels (:obj:`int`, `optional`):
+            Maximum length of the ``labels`` returned list and optionally padding length (see above).
+        pad_to_multiple_of (:obj:`int`, `optional`):
+            If set will pad the sequence to a multiple of the provided value.
+            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
+            7.5 (Volta).
+    """
+
+    processor: Wav2Vec2Processor
+    padding: Union[bool, str] = True
+    max_length: Optional[int] = None
+    max_length_labels: Optional[int] = None
+    pad_to_multiple_of: Optional[int] = None
+    pad_to_multiple_of_labels: Optional[int] = None
+
+    def __call__(
+        self, features: List[Dict[str, Union[List[int], torch.Tensor]]]
+    ) -> Dict[str, torch.Tensor]:
+        # split inputs and labels since they have to be of different lenghts and need
+        # different padding methods
+        input_features = [
+            {"input_values": feature["input_values"]} for feature in features
+        ]
+        label_features = [{"input_ids": feature["labels"]} for feature in features]
+
+        batch = self.processor.pad(
+            input_features,
+            padding=self.padding,
+            max_length=self.max_length,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors="pt",
+        )
+        with self.processor.as_target_processor():
+            labels_batch = self.processor.pad(
+                label_features,
+                padding=self.padding,
+                max_length=self.max_length_labels,
+                pad_to_multiple_of=self.pad_to_multiple_of_labels,
+                return_tensors="pt",
+            )
+
+        # replace padding with -100 to ignore loss correctly
+        labels = labels_batch["input_ids"].masked_fill(
+            labels_batch.attention_mask.ne(1), -100
+        )
+
+        batch["labels"] = labels
+
+        return batch
+
+
+data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)
+# cer_metric = load_metric("./cer")
+
+# def compute_metrics(pred):
+#     pred_logits = pred.predictions
+#     pred_ids = np.argmax(pred_logits, axis=-1)
+
+#     pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id
+
+#     pred_str = processor.batch_decode(pred_ids)
+#     # we do not want to group tokens when computing the metrics
+#     label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
+
+#     cer = cer_metric.compute(predictions=pred_str, references=label_str)
+
+#     return {"cer": cer}
+
+def compute_metrics(pred):
+        pred_logits = pred.predictions
+        pred_ids = np.argmax(pred_logits, axis=-1)
+
+        pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id
+
+        pred_str = tokenizer.batch_decode(pred_ids)
+        # we do not want to group tokens when computing the metrics
+        label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)
+
+        metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()}
+
+        return metrics
+
+model = Wav2Vec2ForCTC.from_pretrained(
+    args.model,
+    attention_dropout=0.1,
+    hidden_dropout=0.1,
+    feat_proj_dropout=0.0,
+    mask_time_prob=0.05,
+    layerdrop=0.1,
+    gradient_checkpointing=True,
+    ctc_loss_reduction="mean",
+    pad_token_id=processor.tokenizer.pad_token_id,
+    vocab_size=len(processor.tokenizer),
+)
+
+if not args.unfreeze:
+    model.freeze_feature_extractor()
+
+training_args = TrainingArguments(
+    output_dir="./finetuned-wav2vec2-xls-r-300m-cantonese/wav2vec2-xls-r-300m-cantonese",
+    group_by_length=True,
+    per_device_train_batch_size=8,
+    gradient_accumulation_steps=2,
+    #evaluation_strategy="no",
+    evaluation_strategy="steps",
+    #evaluation_strategy="epoch",
+    eval_steps=400,
+    #eval_accumulation_steps=60,
+    num_train_epochs=1,
+    fp16=True,
+    fp16_backend="amp",
+    logging_strategy="steps",
+    logging_steps=400,
+    #logging_strategy="epoch",
+    learning_rate=args.lr,
+    warmup_steps=100,
+    save_steps=2376, # every 3 epoch with batch_size 8
+    #save_strategy="epoch",
+    save_total_limit=3,
+    ###################
+    # fp16_full_eval=True,
+    dataloader_num_workers=20,
+)
+
+trainer = Trainer(
+    model=model,
+    data_collator=data_collator,
+    args=training_args,
+    compute_metrics=compute_metrics,
+    train_dataset=common_voice_train,
+    eval_dataset=common_voice_test,
+    tokenizer=processor.feature_extractor,
+)
+trainer.train()

preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4c942c5907b0142b5a50313d966ce9b6ab25576ad72fd231aea7f42a493b103
+size 1278024433

run.sh

@@ -0,0 +1,29 @@
+python run_speech_recognition_ctc.py \
+	--dataset_name="mozilla-foundation/common_voice_8_0" \
+	--model_name_or_path="facebook/wav2vec2-xls-r-300m" \
+	--dataset_config_name="zh-HK" \
+	--output_dir="./" \
+    --cache_dir="../container_0" \
+	--overwrite_output_dir \
+	--num_train_epochs="1" \
+	--per_device_train_batch_size="8" \
+    --per_device_eval_batch_size="1" \
+	--gradient_accumulation_steps="2" \
+	--learning_rate="3e-4" \
+	--warmup_steps="500" \
+	--evaluation_strategy="steps" \
+	--text_column_name="sentence" \
+	--length_column_name="input_length" \
+	--save_steps="10" \
+	--eval_steps="10" \
+	--layerdrop="0.0" \
+	--save_total_limit="3" \
+	--freeze_feature_encoder \
+	--gradient_checkpointing \
+	--fp16 \
+	--group_by_length \
+    --use_auth_token \
+	--push_to_hub \
+	--do_train \
+    --do_eval \
+	--max_duration_in_seconds="3"

run_speech_recognition_ctc.py

@@ -0,0 +1,780 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
+
+import functools
+import json
+import logging
+import os
+import re
+import sys
+import warnings
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
+
+import datasets
+import numpy as np
+import torch
+from datasets import DatasetDict, load_dataset, load_metric
+
+import transformers
+from transformers import (
+    AutoConfig,
+    AutoFeatureExtractor,
+    AutoModelForCTC,
+    AutoProcessor,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    Wav2Vec2Processor,
+    set_seed,
+)
+from transformers.trainer_utils import get_last_checkpoint, is_main_process
+from transformers.utils import check_min_version
+from transformers.utils.versions import require_version
+
+import string
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.17.0.dev0")
+
+require_version("datasets>=1.13.3", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt")
+
+
+logger = logging.getLogger(__name__)
+
+
+def list_field(default=None, metadata=None):
+    return field(default_factory=lambda: default, metadata=metadata)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={"help": "Path to pretrained tokenizer or tokenizer identifier from huggingface.co/models"},
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    freeze_feature_encoder: bool = field(
+        default=True, metadata={"help": "Whether to freeze the feature encoder layers of the model."}
+    )
+    attention_dropout: float = field(
+        default=0.0, metadata={"help": "The dropout ratio for the attention probabilities."}
+    )
+    activation_dropout: float = field(
+        default=0.0, metadata={"help": "The dropout ratio for activations inside the fully connected layer."}
+    )
+    feat_proj_dropout: float = field(default=0.0, metadata={"help": "The dropout ratio for the projected features."})
+    hidden_dropout: float = field(
+        default=0.0,
+        metadata={
+            "help": "The dropout probability for all fully connected layers in the embeddings, encoder, and pooler."
+        },
+    )
+    final_dropout: float = field(
+        default=0.0,
+        metadata={"help": "The dropout probability for the final projection layer."},
+    )
+    mask_time_prob: float = field(
+        default=0.05,
+        metadata={
+            "help": "Probability of each feature vector along the time axis to be chosen as the start of the vector"
+            "span to be masked. Approximately ``mask_time_prob * sequence_length // mask_time_length`` feature"
+            "vectors will be masked along the time axis."
+        },
+    )
+    mask_time_length: int = field(
+        default=10,
+        metadata={"help": "Length of vector span to mask along the time axis."},
+    )
+    mask_feature_prob: float = field(
+        default=0.0,
+        metadata={
+            "help": "Probability of each feature vector along the feature axis to be chosen as the start of the vector"
+            "span to be masked. Approximately ``mask_feature_prob * sequence_length // mask_feature_length`` feature bins will be masked along the time axis."
+        },
+    )
+    mask_feature_length: int = field(
+        default=10,
+        metadata={"help": "Length of vector span to mask along the feature axis."},
+    )
+    layerdrop: float = field(default=0.0, metadata={"help": "The LayerDrop probability."})
+    ctc_loss_reduction: Optional[str] = field(
+        default="mean", metadata={"help": "The way the ctc loss should be reduced. Should be one of 'mean' or 'sum'."}
+    )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+
+    Using `HfArgumentParser` we can turn this class
+    into argparse arguments to be able to specify them on
+    the command line.
+    """
+
+    dataset_name: str = field(
+        metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: str = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_split_name: str = field(
+        default="train+validation",
+        metadata={
+            "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train+validation'"
+        },
+    )
+    eval_split_name: str = field(
+        default="test",
+        metadata={
+            "help": "The name of the evaluation data set split to use (via the datasets library). Defaults to 'test'"
+        },
+    )
+    audio_column_name: str = field(
+        default="audio",
+        metadata={"help": "The name of the dataset column containing the audio data. Defaults to 'audio'"},
+    )
+    text_column_name: str = field(
+        default="text",
+        metadata={"help": "The name of the dataset column containing the text data. Defaults to 'text'"},
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
+            "value if set."
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "value if set."
+        },
+    )
+    chars_to_ignore: List[str] = list_field(
+        default=[",", "?", "¿", ".", "!", "¡", ";", "；", ":", '""', "%", '"', "�", "ʿ", "·", "჻", "~", "՞",
+                 "؟", "،", "।", "॥", "«", "»", "„", "“", "”", "「", "」", "‘", "’", "《", "》", "(", ")", "[", "]",
+                 "{", "}", "=", "`", "_", "+", "<", ">", "…", "–", "°", "´", "ʾ", "‹", "›", "©", "®", "—", "→", "。",
+                 "、", "﹂", "﹁", "‧", "～", "﹏", "，", "｛", "｝", "（", "）", "［", "］", "【", "】", "‥", "〽",
+                 "『", "』", "〝", "〟", "⟨", "⟩", "〜", "：", "！", "？", "♪", "؛", "/", "\\", "º", "−", "^", "ʻ", "ˆ"],
+        metadata={"help": "A list of characters to remove from the transcripts."},
+    )
+    eval_metrics: List[str] = list_field(
+        default=["wer"],
+        metadata={"help": "A list of metrics the model should be evaluated on. E.g. `'wer cer'`"},
+    )
+    max_duration_in_seconds: float = field(
+        default=20.0,
+        metadata={
+            "help": "Filter audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`"
+        },
+    )
+    min_duration_in_seconds: float = field(
+        default=0.0, metadata={"help": "Filter audio files that are shorter than `min_duration_in_seconds` seconds"}
+    )
+    preprocessing_only: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether to only do data preprocessing and skip training. "
+            "This is especially useful when data preprocessing errors out in distributed training due to timeout. "
+            "In this case, one should run the preprocessing in a non-distributed setup with `preprocessing_only=True` "
+            "so that the cached datasets can consequently be loaded in distributed training"
+        },
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": "If :obj:`True`, will use the token generated when running"
+            ":obj:`transformers-cli login` as HTTP bearer authorization for remote files."
+        },
+    )
+    unk_token: str = field(
+        default="[UNK]",
+        metadata={"help": "The unk token for the tokenizer"},
+    )
+    pad_token: str = field(
+        default="[PAD]",
+        metadata={"help": "The padding token for the tokenizer"},
+    )
+    word_delimiter_token: str = field(
+        default="|",
+        metadata={"help": "The word delimiter token for the tokenizer"},
+    )
+    phoneme_language: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The target language that should be used be"
+            " passed to the tokenizer for tokenization. Note that"
+            " this is only relevant if the model classifies the"
+            " input audio to a sequence of phoneme sequences."
+        },
+    )
+
+
+@dataclass
+class DataCollatorCTCWithPadding:
+    """
+    Data collator that will dynamically pad the inputs received.
+    Args:
+        processor (:class:`~transformers.AutoProcessor`)
+            The processor used for proccessing the data.
+        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
+            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
+            among:
+            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+              sequence if provided).
+            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
+              maximum acceptable input length for the model if that argument is not provided.
+            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
+              different lengths).
+        max_length (:obj:`int`, `optional`):
+            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).
+        max_length_labels (:obj:`int`, `optional`):
+            Maximum length of the ``labels`` returned list and optionally padding length (see above).
+        pad_to_multiple_of (:obj:`int`, `optional`):
+            If set will pad the sequence to a multiple of the provided value.
+            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
+            7.5 (Volta).
+    """
+
+    processor: AutoProcessor
+    padding: Union[bool, str] = "longest"
+    pad_to_multiple_of: Optional[int] = None
+    pad_to_multiple_of_labels: Optional[int] = None
+
+    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
+        # split inputs and labels since they have to be of different lenghts and need
+        # different padding methods
+        input_features = [{"input_values": feature["input_values"]} for feature in features]
+        label_features = [{"input_ids": feature["labels"]} for feature in features]
+
+        batch = self.processor.pad(
+            input_features,
+            padding=self.padding,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors="pt",
+        )
+
+        with self.processor.as_target_processor():
+            labels_batch = self.processor.pad(
+                label_features,
+                padding=self.padding,
+                pad_to_multiple_of=self.pad_to_multiple_of_labels,
+                return_tensors="pt",
+            )
+
+        # replace padding with -100 to ignore loss correctly
+        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)
+
+        batch["labels"] = labels
+
+        return batch
+
+
+def create_vocabulary_from_data(
+    datasets: DatasetDict,
+    word_delimiter_token: Optional[str] = None,
+    unk_token: Optional[str] = None,
+    pad_token: Optional[str] = None,
+):
+    # Given training and test labels create vocabulary
+    def extract_all_chars(batch):
+        all_text = " ".join(batch["target_text"])
+        vocab = list(set(all_text))
+        return {"vocab": [vocab], "all_text": [all_text]}
+
+    vocabs = datasets.map(
+        extract_all_chars,
+        batched=True,
+        batch_size=-1,
+        keep_in_memory=True,
+        remove_columns=datasets["train"].column_names,
+    )
+
+    # take union of all unique characters in each dataset
+    vocab_set = functools.reduce(
+        lambda vocab_1, vocab_2: set(vocab_1["vocab"][0]) | set(vocab_2["vocab"][0]), vocabs.values()
+    )
+
+    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocab_set)))}
+
+    # replace white space with delimiter token
+    if word_delimiter_token is not None:
+        vocab_dict[word_delimiter_token] = vocab_dict[" "]
+        del vocab_dict[" "]
+
+    # add unk and pad token
+    if unk_token is not None:
+        vocab_dict[unk_token] = len(vocab_dict)
+
+    if pad_token is not None:
+        vocab_dict[pad_token] = len(vocab_dict)
+
+    return vocab_dict
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # 1. First, let's load the dataset
+    raw_datasets = DatasetDict()
+
+    if training_args.do_train:
+        raw_datasets["train"]  = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            split=data_args.train_split_name,
+            use_auth_token=data_args.use_auth_token,
+        )
+
+        if data_args.audio_column_name not in raw_datasets["train"].column_names:
+            raise ValueError(
+                f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'. "
+                "Make sure to set `--audio_column_name` to the correct audio column - one of "
+                f"{', '.join(raw_datasets['train'].column_names)}."
+            )
+
+        if data_args.text_column_name not in raw_datasets["train"].column_names:
+            raise ValueError(
+                f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. "
+                "Make sure to set `--text_column_name` to the correct text column - one of "
+                f"{', '.join(raw_datasets['train'].column_names)}."
+            )
+
+        if data_args.max_train_samples is not None:
+            raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples))
+
+    if training_args.do_eval:
+        raw_datasets["eval"] = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            split=data_args.eval_split_name,
+            use_auth_token=data_args.use_auth_token,
+        )
+
+        if data_args.max_eval_samples is not None:
+            raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples))
+
+    # 2. We remove some special characters from the datasets
+    # that make training complicated and do not help in transcribing the speech
+    # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
+    # that could be easily picked up by the model
+    chars_to_ignore_regex = (
+        f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
+    )
+    text_column_name = data_args.text_column_name
+
+    def remove_special_characters(batch):
+        if chars_to_ignore_regex is not None:
+            sen = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " "
+            
+            # convert 'D' and 'd' to '啲' if there a 'D' in sentence
+            # hacky stuff, wont work on 'D', 'd' co-occure with normal english words
+            # wont work on multiple 'D'
+            if "d" in sen:
+                if len([c for c in sen if c in string.ascii_lowercase]) == 1:
+                    sen = sen.replace("d", "啲")
+            batch["target_text"] = sen    
+        else:
+            batch["target_text"] = batch[text_column_name].lower() + " "
+        return batch
+
+    with training_args.main_process_first(desc="dataset map special characters removal"):
+        raw_datasets = raw_datasets.map(
+            remove_special_characters,
+            remove_columns=[text_column_name],
+            desc="remove special characters from datasets",
+        )
+
+    # save special tokens for tokenizer
+    word_delimiter_token = data_args.word_delimiter_token
+    unk_token = data_args.unk_token
+    pad_token = data_args.pad_token
+
+    # 3. Next, let's load the config as we might need it to create
+    # the tokenizer
+    # load config
+    config = AutoConfig.from_pretrained(
+        model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
+    )
+
+    # 4. Next, if no tokenizer file is defined,
+    # we create the vocabulary of the model by extracting all unique characters from
+    # the training and evaluation datasets
+    # We need to make sure that only first rank saves vocabulary
+    # make sure all processes wait until vocab is created
+    tokenizer_name_or_path = model_args.tokenizer_name_or_path
+    tokenizer_kwargs = {}
+    if tokenizer_name_or_path is None:
+        # save vocab in training output dir
+        tokenizer_name_or_path = training_args.output_dir
+
+        vocab_file = os.path.join(tokenizer_name_or_path, "vocab.json")
+
+        with training_args.main_process_first():
+            if training_args.overwrite_output_dir and os.path.isfile(vocab_file):
+                os.remove(vocab_file)
+
+        with training_args.main_process_first(desc="dataset map vocabulary creation"):
+            if not os.path.isfile(vocab_file):
+                os.makedirs(tokenizer_name_or_path, exist_ok=True)
+                vocab_dict = create_vocabulary_from_data(
+                    raw_datasets,
+                    word_delimiter_token=word_delimiter_token,
+                    unk_token=unk_token,
+                    pad_token=pad_token,
+                )
+
+                # save vocab dict to be loaded into tokenizer
+                with open(vocab_file, "w") as file:
+                    json.dump(vocab_dict, file)
+
+        # if tokenizer has just been created
+        # it is defined by `tokenizer_class` if present in config else by `model_type`
+        tokenizer_kwargs = {
+            "config": config if config.tokenizer_class is not None else None,
+            "tokenizer_type": config.model_type if config.tokenizer_class is None else None,
+            "unk_token": unk_token,
+            "pad_token": pad_token,
+            "word_delimiter_token": word_delimiter_token,
+        }
+
+    # 5. Now we can instantiate the feature extractor, tokenizer and model
+    # Note for distributed training, the .from_pretrained methods guarantee that only
+    # one local process can concurrently download model & vocab.
+
+    # load feature_extractor and tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_name_or_path,
+        use_auth_token=data_args.use_auth_token,
+        **tokenizer_kwargs,
+    )
+    feature_extractor = AutoFeatureExtractor.from_pretrained(
+        model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
+    )
+
+    # adapt config
+    config.update(
+        {
+            "feat_proj_dropout": model_args.feat_proj_dropout,
+            "attention_dropout": model_args.attention_dropout,
+            "hidden_dropout": model_args.hidden_dropout,
+            "final_dropout": model_args.final_dropout,
+            "mask_time_prob": model_args.mask_time_prob,
+            "mask_time_length": model_args.mask_time_length,
+            "mask_feature_prob": model_args.mask_feature_prob,
+            "mask_feature_length": model_args.mask_feature_length,
+            "gradient_checkpointing": training_args.gradient_checkpointing,
+            "layerdrop": model_args.layerdrop,
+            "ctc_loss_reduction": model_args.ctc_loss_reduction,
+            "pad_token_id": tokenizer.pad_token_id,
+            "vocab_size": len(tokenizer),
+            "activation_dropout": model_args.activation_dropout,
+        }
+    )
+
+    # create model
+    model = AutoModelForCTC.from_pretrained(
+        model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        config=config,
+        use_auth_token=data_args.use_auth_token,
+    )
+
+    # freeze encoder
+    if model_args.freeze_feature_encoder:
+        model.freeze_feature_encoder()
+
+    # 6. Now we preprocess the datasets including remove long audio sample, loading the audio, resampling and normalization
+    # Thankfully, `datasets` takes care of automatically loading and resampling the audio,
+    # so that we just need to set the correct target sampling rate and normalize the input
+    # via the `feature_extractor`
+
+    # make sure that dataset decodes audio with correct sampling rate
+    dataset_sampling_rate = next(iter(raw_datasets.values())).features[data_args.audio_column_name].sampling_rate
+    # print("data sample rate:", dataset_sampling_rate) # 48_000
+    # print("feature sample rate:", feature_extractor.sampling_rate) # 16_000
+
+    # # remove long common voice
+    # def remove_long_common_voicedata(dataset, max_seconds=6):
+    #     #convert pyarrow table to pandas
+    #     dftest= dataset.to_pandas()
+
+    #     #find out length of input_values
+    #     dftest['len']= dftest['target_text'].apply(len)
+
+    #     #for wav2vec training we already resampled to 16khz
+    #     #remove data that is longer than max_seconds (6 seconds ideal)
+    #     maxLength = max_seconds * 16000 
+    #     dftest= dftest[dftest['len']<maxLength]
+    #     dftest = dftest.drop('len', 1)
+
+    #     #convert back to pyarrow table to use in trainer
+    #     dataset= dataset.from_pandas(dftest)
+
+    #     #directly remove do not wait for gc
+    #     del dftest
+    #     return dataset
+
+    # raw_datasets['train'] = remove_long_common_voicedata(raw_datasets['train'], max_seconds=3)
+    # raw_datasets['eval'] = remove_long_common_voicedata(raw_datasets['eval'], max_seconds=3)
+
+
+    # casting column
+    if dataset_sampling_rate != feature_extractor.sampling_rate:
+        raw_datasets = raw_datasets.cast_column(
+            data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate)
+        )
+
+
+    # derive max & min input length for sample rate & max duration
+    max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate
+    min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate
+    audio_column_name = data_args.audio_column_name
+    num_workers = data_args.preprocessing_num_workers
+
+    # `phoneme_language` is only relevant if the model is fine-tuned on phoneme classification
+    phoneme_language = data_args.phoneme_language
+
+    # Preprocessing the datasets.
+    # We need to read the audio files as arrays and tokenize the targets.
+    def prepare_dataset(batch):
+        # load audio
+        sample = batch[audio_column_name]
+
+        inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"])
+        batch["input_values"] = inputs.input_values[0]
+        batch["input_length"] = len(batch["input_values"])
+
+        # encode targets
+        additional_kwargs = {}
+        if phoneme_language is not None:
+            additional_kwargs["phonemizer_lang"] = phoneme_language
+
+        batch["labels"] = tokenizer(batch["target_text"], **additional_kwargs).input_ids
+        return batch
+
+    with training_args.main_process_first(desc="dataset map preprocessing"):
+        vectorized_datasets = raw_datasets.map(
+            prepare_dataset,
+            remove_columns=next(iter(raw_datasets.values())).column_names,
+            num_proc=num_workers,
+            desc="preprocess datasets",
+        )
+
+        def is_audio_in_length_range(length):
+            return length > min_input_length and length < max_input_length
+
+        # filter data that is shorter than min_input_length
+        vectorized_datasets = vectorized_datasets.filter(
+            is_audio_in_length_range,
+            num_proc=num_workers,
+            input_columns=["input_length"],
+        )
+
+    # 7. Next, we can prepare the training.
+    # Let's use word error rate (WER) as our evaluation metric,
+    # instantiate a data collator and the trainer
+
+    # Define evaluation metrics during training, *i.e.* word error rate, character error rate
+    eval_metrics = {metric: load_metric(metric) for metric in data_args.eval_metrics}
+
+    # for large datasets it is advised to run the preprocessing on a
+    # single machine first with ``args.preprocessing_only`` since there will mostly likely
+    # be a timeout when running the script in distributed mode.
+    # In a second step ``args.preprocessing_only`` can then be set to `False` to load the
+    # cached dataset
+    if data_args.preprocessing_only:
+        logger.info(f"Data preprocessing finished. Files cached at {vectorized_datasets.cache_files}")
+        return
+
+    def compute_metrics(pred):
+        pred_logits = pred.predictions
+        pred_ids = np.argmax(pred_logits, axis=-1)
+
+        pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id
+
+        pred_str = tokenizer.batch_decode(pred_ids)
+        # we do not want to group tokens when computing the metrics
+        label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)
+
+        metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()}
+
+        return metrics
+
+    # Now save everything to be able to create a single processor later
+    if is_main_process(training_args.local_rank):
+        # save feature extractor, tokenizer and config
+        feature_extractor.save_pretrained(training_args.output_dir)
+        tokenizer.save_pretrained(training_args.output_dir)
+        config.save_pretrained(training_args.output_dir)
+
+    try:
+        processor = AutoProcessor.from_pretrained(training_args.output_dir)
+    except (OSError, KeyError):
+        warnings.warn(
+            "Loading a processor from a feature extractor config that does not"
+            " include a `processor_class` attribute is deprecated and will be removed in v5. Please add the following "
+            " attribute to your `preprocessor_config.json` file to suppress this warning: "
+            " `'processor_class': 'Wav2Vec2Processor'`",
+            FutureWarning,
+        )
+        processor = Wav2Vec2Processor.from_pretrained(training_args.output_dir)
+
+    # Instantiate custom data collator
+    data_collator = DataCollatorCTCWithPadding(processor=processor)
+
+    # Initialize Trainer
+    trainer = Trainer(
+        model=model,
+        data_collator=data_collator,
+        args=training_args,
+        compute_metrics=compute_metrics,
+        train_dataset=vectorized_datasets["train"] if training_args.do_train else None,
+        eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None,
+        tokenizer=feature_extractor,
+    )
+
+    # 8. Finally, we can start training
+
+    # Training
+    if training_args.do_train:
+
+        # use last checkpoint if exist
+        if last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        elif os.path.isdir(model_args.model_name_or_path):
+            checkpoint = model_args.model_name_or_path
+        else:
+            checkpoint = None
+
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()
+
+        metrics = train_result.metrics
+        max_train_samples = (
+            data_args.max_train_samples
+            if data_args.max_train_samples is not None
+            else len(vectorized_datasets["train"])
+        )
+        metrics["train_samples"] = min(max_train_samples, len(vectorized_datasets["train"]))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    results = {}
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+        metrics = trainer.evaluate()
+        max_eval_samples = (
+            data_args.max_eval_samples if data_args.max_eval_samples is not None else len(vectorized_datasets["eval"])
+        )
+        metrics["eval_samples"] = min(max_eval_samples, len(vectorized_datasets["eval"]))
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    # Write model card and (optionally) push to hub
+    config_name = data_args.dataset_config_name if data_args.dataset_config_name is not None else "na"
+    kwargs = {
+        "finetuned_from": model_args.model_name_or_path,
+        "tasks": "speech-recognition",
+        "tags": ["automatic-speech-recognition", data_args.dataset_name],
+        "dataset_args": f"Config: {config_name}, Training split: {data_args.train_split_name}, Eval split: {data_args.eval_split_name}",
+        "dataset": f"{data_args.dataset_name.upper()} - {config_name.upper()}",
+    }
+    if "common_voice" in data_args.dataset_name:
+        kwargs["language"] = config_name
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:530b8b92ddc86c8636f3692a5bd1f5a725cd44fe2261da30abe50a8651413ae9
+size 2991

vocab.json

@@ -0,0 +1 @@
+{"!": 1, "\"": 2, ",": 3, "-": 4, ".": 5, ":": 6, ";": 7, "?": 8, "a": 9, "b": 10, "c": 11, "d": 12, "e": 13, "f": 14, "g": 15, "h": 16, "i": 17, "j": 18, "k": 19, "l": 20, "m": 21, "n": 22, "o": 23, "p": 24, "q": 25, "r": 26, "s": 27, "t": 28, "u": 29, "v": 30, "w": 31, "x": 32, "y": 33, "z": 34, "~": 35, "·": 36, "–": 37, "—": 38, "“": 39, "”": 40, "…": 41, "‧": 42, "⋯": 43, "⠀": 44, "⻣": 45, "、": 46, "。": 47, "《": 48, "》": 49, "「": 50, "」": 51, "ㄧ": 52, "㗎": 53, "㩒": 54, "㩿": 55, "㪐": 56, "䁪": 57, "䒏": 58, "䒐": 59, "䟴": 60, "䰧": 61, "䱽": 62, "一": 63, "丁": 64, "七": 65, "丈": 66, "三": 67, "上": 68, "下": 69, "不": 70, "丑": 71, "且": 72, "丕": 73, "世": 74, "丘": 75, "丙": 76, "丞": 77, "丟": 78, "両": 79, "並": 80, "丫": 81, "中": 82, "丰": 83, "串": 84, "丶": 85, "丸": 86, "丹": 87, "主": 88, "丼": 89, "乃": 90, "久": 91, "义": 92, "之": 93, "乍": 94, "乎": 95, "乏": 96, "乒": 97, "乓": 98, "乖": 99, "乘": 100, "乙": 101, "乜": 102, "九": 103, "乞": 104, "也": 105, "乳": 106, "乸": 107, "乾": 108, "亂": 109, "了": 110, "予": 111, "事": 112, "二": 113, "于": 114, "云": 115, "互": 116, "五": 117, "井": 118, "些": 119, "亞": 120, "亡": 121, "亢": 122, "交": 123, "亦": 124, "亨": 125, "享": 126, "京": 127, "亭": 128, "亮": 129, "人": 130, "什": 131, "仁": 132, "仆": 133, "仇": 134, "今": 135, "介": 136, "仍": 137, "仔": 138, "仕": 139, "他": 140, "仗": 141, "付": 142, "仙": 143, "仞": 144, "代": 145, "令": 146, "以": 147, "仰": 148, "仱": 149, "仲": 150, "件": 151, "任": 152, "份": 153, "仿": 154, "企": 155, "伊": 156, "伍": 157, "伏": 158, "伐": 159, "休": 160, "伙": 161, "伯": 162, "估": 163, "伴": 164, "伶": 165, "伸": 166, "似": 167, "伽": 168, "佃": 169, "但": 170, "佈": 171, "位": 172, "低": 173, "住": 174, "佐": 175, "佑": 176, "佔": 177, "何": 178, "佗": 179, "余": 180, "佚": 181, "佛": 182, "作": 183, "你": 184, "佢": 185, "佣": 186, "佩": 187, "佬": 188, "佰": 189, "佳": 190, "併": 191, "佻": 192, "使": 193, "侄": 194, "來": 195, "例": 196, "侍": 197, "供": 198, "依": 199, "侮": 200, "侯": 201, "侵": 202, "侶": 203, "便": 204, "係": 205, "促": 206, "俄": 207, "俊": 208, "俎": 209, "俏": 210, "俐": 211, "俗": 212, "俚": 213, "保": 214, "俠": 215, "信": 216, "俬": 217, "修": 218, "俱": 219, "俸": 220, "俹": 221, "俾": 222, "倉": 223, "個": 224, "倍": 225, "們": 226, "倒": 227, "候": 228, "倚": 229, "借": 230, "倦": 231, "倫": 232, "值": 233, "假": 234, "偈": 235, "偉": 236, "偏": 237, "偕": 238, "做": 239, "停": 240, "健": 241, "側": 242, "偶": 243, "偷": 244, "偽": 245, "傅": 246, "傍": 247, "傑": 248, "傘": 249, "備": 250, "傢": 251, "催": 252, "傭": 253, "傲": 254, "傳": 255, "債": 256, "傷": 257, "傻": 258, "傾": 259, "僅": 260, "像": 261, "僑": 262, "僕": 263, "僚": 264, "僧": 265, "僭": 266, "僱": 267, "價": 268, "僻": 269, "儀": 270, "億": 271, "儈": 272, "儍": 273, "儒": 274, "儘": 275, "優": 276, "儬": 277, "儲": 278, "允": 279, "元": 280, "兄": 281, "充": 282, "兆": 283, "兇": 284, "先": 285, "光": 286, "克": 287, "兌": 288, "免": 289, "兒": 290, "兔": 291, "兜": 292, "入": 293, "內": 294, "全": 295, "兩": 296, "八": 297, "公": 298, "六": 299, "兮": 300, "共": 301, "兵": 302, "其": 303, "具": 304, "典": 305, "兼": 306, "内": 307, "冇": 308, "冊": 309, "再": 310, "冒": 311, "冕": 312, "冗": 313, "冚": 314, "冠": 315, "冤": 316, "冧": 317, "冬": 318, "冰": 319, "冷": 320, "准": 321, "凈": 322, "凌": 323, "凍": 324, "凝": 325, "凡": 326, "凰": 327, "凱": 328, "凳": 329, "凶": 330, "凹": 331, "出": 332, "函": 333, "刀": 334, "刁": 335, "刃": 336, "分": 337, "切": 338, "刊": 339, "刑": 340, "划": 341, "列": 342, "初": 343, "判": 344, "別": 345, "刨": 346, "利": 347, "刮": 348, "到": 349, "制": 350, "刷": 351, "券": 352, "刺": 353, "刻": 354, "剃": 355, "則": 356, "前": 357, "剎": 358, "剔": 359, "剛": 360, "剝": 361, "剩": 362, "剪": 363, "副": 364, "割": 365, "創": 366, "剷": 367, "劃": 368, "劇": 369, "劈": 370, "劉": 371, "劊": 372, "劍": 373, "劏": 374, "劑": 375, "劖": 376, "力": 377, "功": 378, "加": 379, "劣": 380, "助": 381, "努": 382, "劫": 383, "勁": 384, "勃": 385, "勇": 386, "勉": 387, "勒": 388, "動": 389, "勘": 390, "務": 391, "勝": 392, "勞": 393, "勢": 394, "勤": 395, "勳": 396, "勵": 397, "勸": 398, "勻": 399, "勾": 400, "勿": 401, "包": 402, "匈": 403, "化": 404, "北": 405, "匙": 406, "匠": 407, "匡": 408, "匯": 409, "匹": 410, "匿": 411, "區": 412, "十": 413, "千": 414, "升": 415, "午": 416, "半": 417, "卑": 418, "卒": 419, "卓": 420, "協": 421, "南": 422, "博": 423, "卜": 424, "占": 425, "卡": 426, "卦": 427, "卧": 428, "印": 429, "危": 430, "即": 431, "卵": 432, "卷": 433, "卸": 434, "卻": 435, "卿": 436, "厄": 437, "厘": 438, "厚": 439, "原": 440, "���": 441, "厭": 442, "厲": 443, "厴": 444, "去": 445, "參": 446, "又": 447, "叉": 448, "及": 449, "友": 450, "反": 451, "叔": 452, "取": 453, "受": 454, "叛": 455, "叢": 456, "口": 457, "古": 458, "句": 459, "另": 460, "叨": 461, "只": 462, "叫": 463, "召": 464, "叭": 465, "叮": 466, "可": 467, "台": 468, "史": 469, "右": 470, "司": 471, "叻": 472, "吃": 473, "各": 474, "合": 475, "吉": 476, "吊": 477, "吋": 478, "同": 479, "名": 480, "后": 481, "吐": 482, "向": 483, "吓": 484, "吖": 485, "君": 486, "吝": 487, "吞": 488, "吟": 489, "吠": 490, "否": 491, "吧": 492, "吩": 493, "含": 494, "吱": 495, "吳": 496, "吵": 497, "吶": 498, "吸": 499, "吹": 500, "吻": 501, "吼": 502, "吽": 503, "吾": 504, "呀": 505, "呂": 506, "呃": 507, "呆": 508, "呈": 509, "告": 510, "呎": 511, "呔": 512, "呢": 513, "周": 514, "呱": 515, "味": 516, "呷": 517, "呻": 518, "呼": 519, "命": 520, "咀": 521, "咁": 522, "咄": 523, "咇": 524, "咋": 525, "和": 526, "咐": 527, "咕": 528, "咖": 529, "咗": 530, "咚": 531, "咦": 532, "咧": 533, "咩": 534, "咪": 535, "咬": 536, "咯": 537, "咳": 538, "咸": 539, "咹": 540, "咽": 541, "咿": 542, "哀": 543, "品": 544, "哂": 545, "哄": 546, "哇": 547, "哈": 548, "哉": 549, "哋": 550, "响": 551, "哎": 552, "員": 553, "哣": 554, "哥": 555, "哦": 556, "哨": 557, "哩": 558, "哪": 559, "哭": 560, "哲": 561, "哺": 562, "哼": 563, "唂": 564, "唇": 565, "唈": 566, "唉": 567, "唎": 568, "唏": 569, "唐": 570, "唔": 571, "唞": 572, "唥": 573, "唧": 574, "唪": 575, "售": 576, "唯": 577, "唱": 578, "唸": 579, "啄": 580, "啅": 581, "商": 582, "啊": 583, "啋": 584, "啍": 585, "問": 586, "啕": 587, "啖": 588, "啜": 589, "啞": 590, "啟": 591, "啡": 592, "啤": 593, "啦": 594, "啩": 595, "啪": 596, "啫": 597, "啱": 598, "啲": 599, "啵": 600, "啼": 601, "啾": 602, "喀": 603, "喂": 604, "喃": 605, "善": 606, "喇": 607, "喉": 608, "喊": 609, "喎": 610, "喐": 611, "喔": 612, "喘": 613, "喙": 614, "喚": 615, "喜": 616, "喝": 617, "喪": 618, "喫": 619, "喬": 620, "單": 621, "喱": 622, "喳": 623, "喺": 624, "喻": 625, "喼": 626, "嗅": 627, "嗇": 628, "嗌": 629, "嗎": 630, "嗒": 631, "嗚": 632, "嗜": 633, "嗟": 634, "嗡": 635, "嗤": 636, "嗦": 637, "嗰": 638, "嗱": 639, "嗲": 640, "嗶": 641, "嗷": 642, "嗽": 643, "嘅": 644, "嘆": 645, "嘈": 646, "嘉": 647, "嘎": 648, "嘔": 649, "嘗": 650, "嘛": 651, "嘜": 652, "嘞": 653, "嘟": 654, "嘢": 655, "嘥": 656, "嘩": 657, "嘲": 658, "嘴": 659, "嘸": 660, "嘻": 661, "嘿": 662, "噁": 663, "噃": 664, "噄": 665, "噉": 666, "噌": 667, "噎": 668, "噏": 669, "噓": 670, "噚": 671, "噤": 672, "器": 673, "噩": 674, "噪": 675, "噬": 676, "噴": 677, "噶": 678, "噹": 679, "嚇": 680, "嚎": 681, "嚐": 682, "嚕": 683, "嚟": 684, "嚡": 685, "嚢": 686, "嚥": 687, "嚨": 688, "嚴": 689, "嚷": 690, "嚼": 691, "嚿": 692, "囉": 693, "囊": 694, "囌": 695, "囍": 696, "囑": 697, "囚": 698, "四": 699, "囝": 700, "回": 701, "因": 702, "囡": 703, "囪": 704, "困": 705, "固": 706, "圃": 707, "圈": 708, "國": 709, "圍": 710, "圑": 711, "園": 712, "圓": 713, "圖": 714, "團": 715, "土": 716, "在": 717, "圭": 718, "地": 719, "圳": 720, "圾": 721, "址": 722, "均": 723, "坊": 724, "坎": 725, "坐": 726, "坑": 727, "坡": 728, "坤": 729, "坦": 730, "坪": 731, "坭": 732, "坳": 733, "坷": 734, "垂": 735, "垃": 736, "型": 737, "垢": 738, "埃": 739, "埋": 740, "城": 741, "埔": 742, "埗": 743, "埞": 744, "域": 745, "埠": 746, "埲": 747, "執": 748, "培": 749, "基": 750, "埼": 751, "堂": 752, "堅": 753, "堆": 754, "堡": 755, "堤": 756, "堪": 757, "報": 758, "場": 759, "堵": 760, "塊": 761, "塑": 762, "塔": 763, "塗": 764, "塘": 765, "塞": 766, "塢": 767, "填": 768, "塱": 769, "塵": 770, "塾": 771, "墀": 772, "境": 773, "墅": 774, "墊": 775, "墓": 776, "墜": 777, "增": 778, "墟": 779, "墨": 780, "墩": 781, "墮": 782, "墳": 783, "壁": 784, "壆": 785, "壇": 786, "壓": 787, "壘": 788, "壞": 789, "壟": 790, "壩": 791, "士": 792, "壯": 793, "壹": 794, "壺": 795, "壽": 796, "夏": 797, "夕": 798, "外": 799, "多": 800, "夜": 801, "夠": 802, "夢": 803, "夥": 804, "大": 805, "天": 806, "太": 807, "夫": 808, "央": 809, "失": 810, "夷": 811, "夾": 812, "奀": 813, "奄": 814, "奇": 815, "奈": 816, "奉": 817, "奏": 818, "契": 819, "奔": 820, "奕": 821, "套": 822, "奚": 823, "奧": 824, "奪": 825, "奮": 826, "女": 827, "奴": 828, "奶": 829, "奸": 830, "她": 831, "好": 832, "如": 833, "妄": 834, "妒": 835, "妓": 836, "妙": 837, "妝": 838, "妥": 839, "妨": 840, "妳": 841, "妹": 842, "妻": 843, "妾": 844, "姆": 845, "姊": 846, "始": 847, "姐": 848, "姑": 849, "姓": 850, "委": 851, "姜": 852, "姣": 853, "姦": 854, "姨": 855, "姬": 856, "姻": 857, "姿": 858, "威": 859, "娃": 860, "娘": 861, "娛": 862, "娜": 863, "娥": 864, "娶": 865, "婆": 866, "婚": 867, "婢": 868, "婦": 869, "婷": 870, "媒": 871, "媚": 872, "媽": 873, "媾": 874, "嫁": 875, "嫂": 876, "嫉": 877, "嫌": 878, "嫩": 879, "嫪": 880, "嫲": 881, "嫻": 882, "嬉": 883, "嬌": 884, "嬲": 885, "嬸": 886, "子": 887, "孔": 888, "孕": 889, "孖": 890, "字": 891, "存": 892, "孚": 893, "孝": 894, "孟": 895, "季": 896, "孤": 897, "孥": 898, "孩": 899, "孫": 900, "孭": 901, "孰": 902, "孱": 903, "學": 904, "孽": 905, "它": 906, "宅": 907, "宇": 908, "守": 909, "安": 910, "宋": 911, "完": 912, "宏": 913, "宗": 914, "官": 915, "宙": 916, "定": 917, "宛": 918, "宜": 919, "客": 920, "宣": 921, "室": 922, "宮": 923, "宰": 924, "害": 925, "宴": 926, "宵": 927, "家": 928, "宸": 929, "容": 930, "宿": 931, "寂": 932, "寃": 933, "寄": 934, "寅": 935, "密": 936, "寇": 937, "富": 938, "寒": 939, "寓": 940, "寞": 941, "察": 942, "寡": 943, "寢": 944, "實": 945, "寧": 946, "寨": 947, "審": 948, "寫": 949, "寬": 950, "寮": 951, "寰": 952, "寵": 953, "寶": 954, "寸": 955, "寺": 956, "封": 957, "射": 958, "將": 959, "專": 960, "尊": 961, "尋": 962, "對": 963, "導": 964, "小": 965, "少": 966, "尖": 967, "尚": 968, "尤": 969, "尬": 970, "就": 971, "尷": 972, "尺": 973, "尼": 974, "尾": 975, "尿": 976, "局": 977, "屁": 978, "居": 979, "屆": 980, "屈": 981, "屋": 982, "屌": 983, "屍": 984, "屎": 985, "屏": 986, "屐": 987, "屑": 988, "展": 989, "屙": 990, "屠": 991, "層": 992, "履": 993, "屬": 994, "屯": 995, "山": 996, "屹": 997, "岀": 998, "岑": 999, "岡": 1000, "岩": 1001, "岬": 1002, "岳": 1003, "岸": 1004, "峒": 1005, "峨": 1006, "峯": 1007, "峰": 1008, "島": 1009, "峻": 1010, "峽": 1011, "崆": 1012, "崇": 1013, "崎": 1014, "崗": 1015, "崙": 1016, "崧": 1017, "崩": 1018, "嵋": 1019, "嵌": 1020, "嶄": 1021, "嶙": 1022, "嶺": 1023, "嶼": 1024, "巉": 1025, "巒": 1026, "川": 1027, "州": 1028, "巡": 1029, "巢": 1030, "工": 1031, "左": 1032, "巧": 1033, "巨": 1034, "巫": 1035, "差": 1036, "己": 1037, "已": 1038, "巴": 1039, "巷": 1040, "巾": 1041, "巿": 1042, "市": 1043, "布": 1044, "帆": 1045, "希": 1046, "帖": 1047, "帚": 1048, "帝": 1049, "帥": 1050, "師": 1051, "席": 1052, "帳": 1053, "帶": 1054, "常": 1055, "帽": 1056, "幃": 1057, "幅": 1058, "幕": 1059, "幟": 1060, "幡": 1061, "幢": 1062, "幣": 1063, "幫": 1064, "干": 1065, "平": 1066, "年": 1067, "幸": 1068, "幹": 1069, "幻": 1070, "幼": 1071, "幽": 1072, "幾": 1073, "庇": 1074, "床": 1075, "序": 1076, "底": 1077, "店": 1078, "庚": 1079, "府": 1080, "度": 1081, "座": 1082, "庫": 1083, "庭": 1084, "庵": 1085, "庶": 1086, "康": 1087, "庸": 1088, "庹": 1089, "廁": 1090, "廂": 1091, "廈": 1092, "廉": 1093, "廊": 1094, "廖": 1095, "廚": 1096, "廟": 1097, "廠": 1098, "廢": 1099, "廣": 1100, "廬": 1101, "廳": 1102, "延": 1103, "廷": 1104, "建": 1105, "廿": 1106, "弄": 1107, "弊": 1108, "弍": 1109, "式": 1110, "弓": 1111, "引": 1112, "弛": 1113, "弟": 1114, "弱": 1115, "張": 1116, "強": 1117, "弸": 1118, "强": 1119, "弼": 1120, "彈": 1121, "彌": 1122, "彎": 1123, "彗": 1124, "彙": 1125, "形": 1126, "彤": 1127, "彥": 1128, "彩": 1129, "彪": 1130, "彬": 1131, "彭": 1132, "影": 1133, "彷": 1134, "役": 1135, "彼": 1136, "彿": 1137, "往": 1138, "征": 1139, "待": 1140, "徇": 1141, "很": 1142, "徊": 1143, "律": 1144, "後": 1145, "徐": 1146, "徑": 1147, "徒": 1148, "得": 1149, "徘": 1150, "從": 1151, "御": 1152, "復": 1153, "循": 1154, "微": 1155, "徵": 1156, "德": 1157, "徹": 1158, "徽": 1159, "心": 1160, "必": 1161, "忌": 1162, "忍": 1163, "志": 1164, "忘": 1165, "忙": 1166, "忟": 1167, "忠": 1168, "快": 1169, "念": 1170, "忽": 1171, "忿": 1172, "怎": 1173, "怒": 1174, "怕": 1175, "怖": 1176, "思": 1177, "怡": 1178, "急": 1179, "怦": 1180, "性": 1181, "怨": 1182, "怪": 1183, "怯": 1184, "恃": 1185, "恆": 1186, "恐": 1187, "恒": 1188, "恕": 1189, "恙": 1190, "恢": 1191, "恣": 1192, "恤": 1193, "恥": 1194, "恨": 1195, "恩": 1196, "恭": 1197, "息": 1198, "恰": 1199, "悄": 1200, "悅": 1201, "悉": 1202, "悒": 1203, "悔": 1204, "悖": 1205, "悗": 1206, "悟": 1207, "悠": 1208, "患": 1209, "您": 1210, "悲": 1211, "悶": 1212, "悽": 1213, "情": 1214, "惇": 1215, "惑": 1216, "惘": 1217, "惜": 1218, "惟": 1219, "惠": 1220, "惡": 1221, "惦": 1222, "惰": 1223, "惱": 1224, "想": 1225, "惶": 1226, "惹": 1227, "愁": 1228, "愈": 1229, "愉": 1230, "意": 1231, "愚": 1232, "愛": 1233, "感": 1234, "愧": 1235, "愿": 1236, "慈": 1237, "態": 1238, "慌": 1239, "慎": 1240, "慕": 1241, "慘": 1242, "慚": 1243, "慢": 1244, "慣": 1245, "慤": 1246, "慧": 1247, "慨": 1248, "慮": 1249, "慰": 1250, "慳": 1251, "慶": 1252, "慷": 1253, "慾": 1254, "憂": 1255, "憎": 1256, "憐": 1257, "憑": 1258, "憚": 1259, "憤": 1260, "憧": 1261, "憩": 1262, "憫": 1263, "憬": 1264, "憲": 1265, "憶": 1266, "憾": 1267, "懂": 1268, "懇": 1269, "應": 1270, "懊": 1271, "懞": 1272, "懣": 1273, "懵": 1274, "懶": 1275, "懷": 1276, "懸": 1277, "懺": 1278, "懼": 1279, "懿": 1280, "戀": 1281, "戇": 1282, "戊": 1283, "戎": 1284, "成": 1285, "我": 1286, "戒": 1287, "戕": 1288, "或": 1289, "戚": 1290, "戟": 1291, "戥": 1292, "截": 1293, "戯": 1294, "戰": 1295, "戲": 1296, "戴": 1297, "戶": 1298, "戽": 1299, "戾": 1300, "房": 1301, "所": 1302, "扁": 1303, "扂": 1304, "扇": 1305, "手": 1306, "才": 1307, "扎": 1308, "扑": 1309, "扒": 1310, "打": 1311, "托": 1312, "扣": 1313, "扭": 1314, "扮": 1315, "扯": 1316, "扶": 1317, "批": 1318, "扻": 1319, "扼": 1320, "找": 1321, "承": 1322, "技": 1323, "抄": 1324, "抆": 1325, "把": 1326, "抑": 1327, "抓": 1328, "投": 1329, "抖": 1330, "抗": 1331, "折": 1332, "抦": 1333, "抬": 1334, "抱": 1335, "抵": 1336, "抹": 1337, "押": 1338, "抽": 1339, "抿": 1340, "拂": 1341, "拃": 1342, "拆": 1343, "拉": 1344, "拋": 1345, "拌": 1346, "拍": 1347, "拎": 1348, "拐": 1349, "拒": 1350, "拓": 1351, "拔": 1352, "拖": 1353, "拗": 1354, "拘": 1355, "拙": 1356, "招": 1357, "拜": 1358, "括": 1359, "拮": 1360, "拯": 1361, "拱": 1362, "拳": 1363, "拼": 1364, "拾": 1365, "拿": 1366, "持": 1367, "指": 1368, "挈": 1369, "按": 1370, "挑": 1371, "挖": 1372, "挨": 1373, "挪": 1374, "挫": 1375, "振": 1376, "挺": 1377, "挽": 1378, "挾": 1379, "捉": 1380, "捋": 1381, "捌": 1382, "捍": 1383, "捏": 1384, "捐": 1385, "捕": 1386, "捧": 1387, "捨": 1388, "捩": 1389, "据": 1390, "捱": 1391, "捲": 1392, "捶": 1393, "捷": 1394, "捹": 1395, "捺": 1396, "捽": 1397, "掂": 1398, "掃": 1399, "掅": 1400, "授": 1401, "掉": 1402, "掌": 1403, "排": 1404, "掕": 1405, "掗": 1406, "掘": 1407, "掙": 1408, "掛": 1409, "掟": 1410, "掠": 1411, "採": 1412, "探": 1413, "掣": 1414, "接": 1415, "控": 1416, "推": 1417, "掩": 1418, "措": 1419, "揀": 1420, "揇": 1421, "揈": 1422, "揉": 1423, "描": 1424, "提": 1425, "插": 1426, "揗": 1427, "揚": 1428, "換": 1429, "揞": 1430, "握": 1431, "揣": 1432, "揦": 1433, "揩": 1434, "揪": 1435, "揭": 1436, "揮": 1437, "揳": 1438, "援": 1439, "揸": 1440, "揼": 1441, "揾": 1442, "損": 1443, "搏": 1444, "搓": 1445, "搔": 1446, "搖": 1447, "搗": 1448, "搜": 1449, "搞": 1450, "搣": 1451, "搬": 1452, "搭": 1453, "搵": 1454, "搶": 1455, "搽": 1456, "摑": 1457, "摘": 1458, "摙": 1459, "摞": 1460, "摧": 1461, "摩": 1462, "摯": 1463, "摳": 1464, "摷": 1465, "摸": 1466, "摺": 1467, "摻": 1468, "撇": 1469, "撈": 1470, "撐": 1471, "撒": 1472, "撓": 1473, "撕": 1474, "撚": 1475, "撞": 1476, "撤": 1477, "撥": 1478, "撩": 1479, "撫": 1480, "撬": 1481, "播": 1482, "撮": 1483, "撲": 1484, "撳": 1485, "撻": 1486, "撼": 1487, "撿": 1488, "擁": 1489, "擂": 1490, "擅": 1491, "擇": 1492, "擊": 1493, "擋": 1494, "操": 1495, "擎": 1496, "擏": 1497, "擒": 1498, "擔": 1499, "擘": 1500, "據": 1501, "擤": 1502, "擦": 1503, "擬": 1504, "擰": 1505, "擲": 1506, "擳": 1507, "擴": 1508, "擷": 1509, "擸": 1510, "擺": 1511, "擾": 1512, "攀": 1513, "攋": 1514, "攏": 1515, "攔": 1516, "攘": 1517, "攝": 1518, "攞": 1519, "攣": 1520, "攤": 1521, "攪": 1522, "攬": 1523, "支": 1524, "攰": 1525, "收": 1526, "攸": 1527, "改": 1528, "攻": 1529, "放": 1530, "政": 1531, "故": 1532, "效": 1533, "敏": 1534, "救": 1535, "敗": 1536, "敘": 1537, "教": 1538, "敝": 1539, "敢": 1540, "散": 1541, "敦": 1542, "敬": 1543, "敲": 1544, "整": 1545, "敵": 1546, "敷": 1547, "數": 1548, "斂": 1549, "斃": 1550, "文": 1551, "斐": 1552, "斑": 1553, "斕": 1554, "斗": 1555, "料": 1556, "斜": 1557, "斟": 1558, "斤": 1559, "斧": 1560, "斬": 1561, "斯": 1562, "新": 1563, "斷": 1564, "方": 1565, "於": 1566, "施": 1567, "旁": 1568, "旅": 1569, "旋": 1570, "旌": 1571, "族": 1572, "旗": 1573, "既": 1574, "旣": 1575, "日": 1576, "旦": 1577, "旨": 1578, "早": 1579, "旬": 1580, "旭": 1581, "旮": 1582, "旯": 1583, "旱": 1584, "旳": 1585, "旺": 1586, "昂": 1587, "昃": 1588, "昆": 1589, "昇": 1590, "昌": 1591, "明": 1592, "昏": 1593, "昐": 1594, "易": 1595, "昔": 1596, "星": 1597, "映": 1598, "春": 1599, "昧": 1600, "昨": 1601, "昭": 1602, "是": 1603, "昺": 1604, "時": 1605, "晃": 1606, "晉": 1607, "晌": 1608, "晏": 1609, "晒": 1610, "晚": 1611, "晝": 1612, "晤": 1613, "晨": 1614, "普": 1615, "景": 1616, "晴": 1617, "晶": 1618, "智": 1619, "晾": 1620, "暇": 1621, "暈": 1622, "暉": 1623, "暑": 1624, "暖": 1625, "暗": 1626, "暢": 1627, "暨": 1628, "暫": 1629, "暮": 1630, "暴": 1631, "暸": 1632, "曆": 1633, "曉": 1634, "曖": 1635, "曙": 1636, "曜": 1637, "曦": 1638, "曬": 1639, "曱": 1640, "曲": 1641, "曳": 1642, "更": 1643, "書": 1644, "曹": 1645, "曼": 1646, "曾": 1647, "替": 1648, "最": 1649, "會": 1650, "月": 1651, "有": 1652, "朋": 1653, "服": 1654, "朕": 1655, "朗": 1656, "望": 1657, "朝": 1658, "期": 1659, "朦": 1660, "朧": 1661, "木": 1662, "未": 1663, "末": 1664, "本": 1665, "札": 1666, "朱": 1667, "朴": 1668, "朵": 1669, "朽": 1670, "杆": 1671, "杉": 1672, "李": 1673, "杏": 1674, "材": 1675, "村": 1676, "杖": 1677, "杜": 1678, "杞": 1679, "束": 1680, "来": 1681, "杭": 1682, "杮": 1683, "杯": 1684, "杰": 1685, "東": 1686, "杷": 1687, "松": 1688, "板": 1689, "枇": 1690, "枉": 1691, "析": 1692, "枕": 1693, "林": 1694, "枚": 1695, "果": 1696, "枝": 1697, "枯": 1698, "枱": 1699, "架": 1700, "柄": 1701, "柏": 1702, "某": 1703, "柑": 1704, "柒": 1705, "染": 1706, "柔": 1707, "柚": 1708, "柞": 1709, "查": 1710, "柬": 1711, "柯": 1712, "柱": 1713, "柳": 1714, "柴": 1715, "柵": 1716, "柺": 1717, "柿": 1718, "栗": 1719, "校": 1720, "栢": 1721, "核": 1722, "根": 1723, "格": 1724, "栽": 1725, "桂": 1726, "桃": 1727, "桅": 1728, "案": 1729, "桌": 1730, "桐": 1731, "桑": 1732, "桔": 1733, "桶": 1734, "桿": 1735, "梁": 1736, "梅": 1737, "梓": 1738, "梗": 1739, "梘": 1740, "條": 1741, "梧": 1742, "梨": 1743, "梭": 1744, "梯": 1745, "械": 1746, "梳": 1747, "梵": 1748, "棄": 1749, "棉": 1750, "棋": 1751, "棍": 1752, "棒": 1753, "棕": 1754, "棖": 1755, "棗": 1756, "棘": 1757, "棚": 1758, "棟": 1759, "棠": 1760, "棧": 1761, "森": 1762, "棲": 1763, "棺": 1764, "椅": 1765, "植": 1766, "椏": 1767, "椒": 1768, "椰": 1769, "楂": 1770, "楊": 1771, "楋": 1772, "楓": 1773, "楚": 1774, "楣": 1775, "業": 1776, "極": 1777, "概": 1778, "榆": 1779, "榔": 1780, "榕": 1781, "榚": 1782, "榛": 1783, "榜": 1784, "榨": 1785, "榮": 1786, "榴": 1787, "構": 1788, "槍": 1789, "槐": 1790, "槤": 1791, "槽": 1792, "樂": 1793, "樊": 1794, "樑": 1795, "樓": 1796, "標": 1797, "樞": 1798, "樟": 1799, "模": 1800, "樣": 1801, "樸": 1802, "樹": 1803, "樺": 1804, "樽": 1805, "橋": 1806, "橘": 1807, "橙": 1808, "機": 1809, "橡": 1810, "橢": 1811, "橫": 1812, "檀": 1813, "檔": 1814, "檢": 1815, "檬": 1816, "檯": 1817, "檳": 1818, "檸": 1819, "檻": 1820, "櫃": 1821, "櫈": 1822, "櫚": 1823, "櫸": 1824, "櫻": 1825, "欄": 1826, "權": 1827, "欖": 1828, "欠": 1829, "次": 1830, "欣": 1831, "欲": 1832, "欺": 1833, "欽": 1834, "款": 1835, "歇": 1836, "歉": 1837, "歌": 1838, "歎": 1839, "歐": 1840, "歛": 1841, "歡": 1842, "止": 1843, "正": 1844, "此": 1845, "步": 1846, "武": 1847, "歧": 1848, "歪": 1849, "歲": 1850, "歷": 1851, "歸": 1852, "歹": 1853, "死": 1854, "殄": 1855, "殆": 1856, "殊": 1857, "殖": 1858, "殘": 1859, "殮": 1860, "殯": 1861, "段": 1862, "殷": 1863, "殺": 1864, "殼": 1865, "殿": 1866, "毀": 1867, "毅": 1868, "毋": 1869, "母": 1870, "每": 1871, "毒": 1872, "毓": 1873, "比": 1874, "毛": 1875, "毡": 1876, "毫": 1877, "氈": 1878, "氏": 1879, "民": 1880, "氓": 1881, "氛": 1882, "氣": 1883, "氧": 1884, "氯": 1885, "水": 1886, "永": 1887, "氹": 1888, "汀": 1889, "汁": 1890, "求": 1891, "汕": 1892, "汗": 1893, "汝": 1894, "江": 1895, "池": 1896, "污": 1897, "汪": 1898, "汰": 1899, "汶": 1900, "決": 1901, "汽": 1902, "沃": 1903, "沈": 1904, "沉": 1905, "沐": 1906, "沒": 1907, "沖": 1908, "沙": 1909, "沛": 1910, "沫": 1911, "沮": 1912, "沱": 1913, "河": 1914, "油": 1915, "治": 1916, "沽": 1917, "沾": 1918, "沿": 1919, "況": 1920, "泄": 1921, "泉": 1922, "泊": 1923, "泌": 1924, "泓": 1925, "法": 1926, "泛": 1927, "泡": 1928, "波": 1929, "泣": 1930, "泥": 1931, "注": 1932, "泮": 1933, "泰": 1934, "泱": 1935, "泳": 1936, "洋": 1937, "洗": 1938, "洛": 1939, "洞": 1940, "津": 1941, "洪": 1942, "洱": 1943, "洲": 1944, "洶": 1945, "活": 1946, "洽": 1947, "派": 1948, "流": 1949, "浙": 1950, "浚": 1951, "浣": 1952, "浦": 1953, "浩": 1954, "浪": 1955, "浮": 1956, "浴": 1957, "海": 1958, "浸": 1959, "涂": 1960, "消": 1961, "涉": 1962, "涌": 1963, "涕": 1964, "涯": 1965, "液": 1966, "涵": 1967, "涷": 1968, "涼": 1969, "淆": 1970, "淋": 1971, "淒": 1972, "淘": 1973, "淚": 1974, "淡": 1975, "淥": 1976, "淨": 1977, "淩": 1978, "淪": 1979, "淫": 1980, "淮": 1981, "深": 1982, "混": 1983, "淸": 1984, "淺": 1985, "添": 1986, "清": 1987, "減": 1988, "渝": 1989, "渠": 1990, "渡": 1991, "渣": 1992, "渦": 1993, "温": 1994, "測": 1995, "渭": 1996, "港": 1997, "渴": 1998, "游": 1999, "渺": 2000, "渾": 2001, "湃": 2002, "湖": 2003, "湘": 2004, "湛": 2005, "湧": 2006, "湯": 2007, "溋": 2008, "源": 2009, "準": 2010, "溜": 2011, "溝": 2012, "溢": 2013, "溥": 2014, "溪": 2015, "溫": 2016, "溯": 2017, "溶": 2018, "滂": 2019, "滄": 2020, "滅": 2021, "滋": 2022, "滌": 2023, "滑": 2024, "滔": 2025, "滘": 2026, "滙": 2027, "滯": 2028, "滴": 2029, "滷": 2030, "滾": 2031, "滿": 2032, "漁": 2033, "漂": 2034, "漆": 2035, "漉": 2036, "漏": 2037, "漓": 2038, "演": 2039, "漚": 2040, "漠": 2041, "漢": 2042, "漫": 2043, "漬": 2044, "漱": 2045, "漲": 2046, "漸": 2047, "漾": 2048, "漿": 2049, "潑": 2050, "潔": 2051, "潘": 2052, "潛": 2053, "潤": 2054, "潭": 2055, "潮": 2056, "潰": 2057, "潲": 2058, "潷": 2059, "潺": 2060, "澀": 2061, "澄": 2062, "澍": 2063, "澎": 2064, "澡": 2065, "澤": 2066, "澩": 2067, "澱": 2068, "澳": 2069, "激": 2070, "濃": 2071, "濕": 2072, "濛": 2073, "濟": 2074, "濠": 2075, "濤": 2076, "濫": 2077, "濱": 2078, "濺": 2079, "濾": 2080, "瀉": 2081, "瀏": 2082, "瀚": 2083, "瀝": 2084, "瀟": 2085, "瀨": 2086, "瀾": 2087, "灌": 2088, "灑": 2089, "灘": 2090, "灣": 2091, "火": 2092, "灰": 2093, "灶": 2094, "灼": 2095, "災": 2096, "炆": 2097, "炊": 2098, "炎": 2099, "炒": 2100, "炕": 2101, "炙": 2102, "炭": 2103, "炮": 2104, "炳": 2105, "炸": 2106, "為": 2107, "烈": 2108, "烏": 2109, "烘": 2110, "烙": 2111, "烚": 2112, "烟": 2113, "烤": 2114, "烹": 2115, "烽": 2116, "焉": 2117, "焗": 2118, "焚": 2119, "無": 2120, "焦": 2121, "然": 2122, "煉": 2123, "煎": 2124, "煖": 2125, "煙": 2126, "煞": 2127, "煤": 2128, "照": 2129, "煨": 2130, "煩": 2131, "煮": 2132, "煲": 2133, "煽": 2134, "熄": 2135, "熊": 2136, "熒": 2137, "熔": 2138, "熙": 2139, "熟": 2140, "熨": 2141, "熬": 2142, "熱": 2143, "熾": 2144, "燃": 2145, "燈": 2146, "燉": 2147, "燒": 2148, "燕": 2149, "燙": 2150, "燜": 2151, "營": 2152, "燥": 2153, "燭": 2154, "燴": 2155, "燶": 2156, "爆": 2157, "爐": 2158, "爛": 2159, "爪": 2160, "爬": 2161, "爭": 2162, "爲": 2163, "爵": 2164, "父": 2165, "爸": 2166, "爹": 2167, "爺": 2168, "爽": 2169, "爾": 2170, "牀": 2171, "牆": 2172, "片": 2173, "版": 2174, "牌": 2175, "牘": 2176, "牙": 2177, "牛": 2178, "牡": 2179, "牢": 2180, "牧": 2181, "物": 2182, "牯": 2183, "牲": 2184, "特": 2185, "牽": 2186, "犀": 2187, "犁": 2188, "犧": 2189, "犬": 2190, "犯": 2191, "狀": 2192, "狂": 2193, "狄": 2194, "狐": 2195, "狗": 2196, "狠": 2197, "狡": 2198, "狩": 2199, "狸": 2200, "狹": 2201, "狼": 2202, "猄": 2203, "猛": 2204, "猜": 2205, "猴": 2206, "猶": 2207, "猾": 2208, "猿": 2209, "獄": 2210, "獅": 2211, "獎": 2212, "獠": 2213, "獨": 2214, "獲": 2215, "獵": 2216, "獸": 2217, "獻": 2218, "玄": 2219, "率": 2220, "玉": 2221, "王": 2222, "玟": 2223, "玩": 2224, "玫": 2225, "玲": 2226, "玻": 2227, "珀": 2228, "珊": 2229, "珍": 2230, "珏": 2231, "珒": 2232, "珠": 2233, "班": 2234, "現": 2235, "球": 2236, "理": 2237, "琉": 2238, "琚": 2239, "琛": 2240, "琦": 2241, "琳": 2242, "琴": 2243, "琵": 2244, "琶": 2245, "瑆": 2246, "瑕": 2247, "瑙": 2248, "瑚": 2249, "瑜": 2250, "瑞": 2251, "瑟": 2252, "瑤": 2253, "瑧": 2254, "瑪": 2255, "瑰": 2256, "璀": 2257, "璃": 2258, "璇": 2259, "璈": 2260, "璉": 2261, "璐": 2262, "璟": 2263, "璧": 2264, "璨": 2265, "環": 2266, "璵": 2267, "璽": 2268, "瓊": 2269, "瓏": 2270, "瓜": 2271, "瓦": 2272, "瓶": 2273, "甘": 2274, "甚": 2275, "甜": 2276, "生": 2277, "產": 2278, "甥": 2279, "用": 2280, "甩": 2281, "甫": 2282, "田": 2283, "由": 2284, "甲": 2285, "申": 2286, "甴": 2287, "男": 2288, "甸": 2289, "畀": 2290, "畋": 2291, "界": 2292, "畏": 2293, "畐": 2294, "畔": 2295, "留": 2296, "畜": 2297, "畝": 2298, "畢": 2299, "略": 2300, "番": 2301, "畫": 2302, "異": 2303, "當": 2304, "畿": 2305, "疆": 2306, "疇": 2307, "疊": 2308, "疏": 2309, "疑": 2310, "疚": 2311, "疤": 2312, "疫": 2313, "疲": 2314, "疵": 2315, "疹": 2316, "疼": 2317, "疾": 2318, "病": 2319, "症": 2320, "痕": 2321, "痛": 2322, "痢": 2323, "痧": 2324, "痰": 2325, "痱": 2326, "痴": 2327, "痺": 2328, "痾": 2329, "瘀": 2330, "瘁": 2331, "瘋": 2332, "瘌": 2333, "瘓": 2334, "瘟": 2335, "瘡": 2336, "瘦": 2337, "療": 2338, "癆": 2339, "癌": 2340, "癡": 2341, "癢": 2342, "癩": 2343, "癮": 2344, "癱": 2345, "癲": 2346, "登": 2347, "發": 2348, "白": 2349, "百": 2350, "皂": 2351, "的": 2352, "皆": 2353, "皇": 2354, "皓": 2355, "皚": 2356, "皮": 2357, "皺": 2358, "盃": 2359, "盅": 2360, "盆": 2361, "盈": 2362, "益": 2363, "盏": 2364, "盒": 2365, "盔": 2366, "盛": 2367, "盜": 2368, "盞": 2369, "盟": 2370, "盡": 2371, "監": 2372, "盤": 2373, "盧": 2374, "盪": 2375, "目": 2376, "盲": 2377, "直": 2378, "相": 2379, "盼": 2380, "盾": 2381, "省": 2382, "眉": 2383, "看": 2384, "眞": 2385, "真": 2386, "眠": 2387, "眨": 2388, "眯": 2389, "眶": 2390, "眷": 2391, "眸": 2392, "眼": 2393, "眾": 2394, "着": 2395, "睄": 2396, "睇": 2397, "睏": 2398, "睛": 2399, "睜": 2400, "睡": 2401, "督": 2402, "睥": 2403, "睦": 2404, "睨": 2405, "睬": 2406, "睹": 2407, "睿": 2408, "瞄": 2409, "瞅": 2410, "瞌": 2411, "瞓": 2412, "瞞": 2413, "瞬": 2414, "瞭": 2415, "矛": 2416, "知": 2417, "矩": 2418, "短": 2419, "矮": 2420, "石": 2421, "砂": 2422, "砌": 2423, "砍": 2424, "砒": 2425, "研": 2426, "砰": 2427, "砲": 2428, "破": 2429, "砵": 2430, "砸": 2431, "硤": 2432, "硬": 2433, "碇": 2434, "碉": 2435, "碌": 2436, "碎": 2437, "碑": 2438, "碗": 2439, "碘": 2440, "碟": 2441, "碧": 2442, "碰": 2443, "確": 2444, "碼": 2445, "磅": 2446, "磐": 2447, "磚": 2448, "磡": 2449, "磨": 2450, "磯": 2451, "礎": 2452, "礙": 2453, "礦": 2454, "礫": 2455, "示": 2456, "社": 2457, "祈": 2458, "祐": 2459, "祖": 2460, "祝": 2461, "神": 2462, "祟": 2463, "祠": 2464, "祥": 2465, "票": 2466, "祭": 2467, "祿": 2468, "禁": 2469, "禍": 2470, "福": 2471, "禡": 2472, "禧": 2473, "禪": 2474, "禮": 2475, "禱": 2476, "禽": 2477, "禾": 2478, "秀": 2479, "私": 2480, "秅": 2481, "秉": 2482, "秋": 2483, "科": 2484, "秒": 2485, "秘": 2486, "租": 2487, "秤": 2488, "秦": 2489, "秧": 2490, "秩": 2491, "移": 2492, "稀": 2493, "稅": 2494, "稈": 2495, "程": 2496, "稍": 2497, "稔": 2498, "稚": 2499, "稟": 2500, "稠": 2501, "種": 2502, "稱": 2503, "稻": 2504, "稿": 2505, "穀": 2506, "穌": 2507, "積": 2508, "穎": 2509, "穗": 2510, "穢": 2511, "穩": 2512, "穫": 2513, "穴": 2514, "究": 2515, "空": 2516, "穿": 2517, "突": 2518, "窄": 2519, "窒": 2520, "窗": 2521, "窠": 2522, "窩": 2523, "窮": 2524, "窰": 2525, "窿": 2526, "竄": 2527, "竅": 2528, "竇": 2529, "竊": 2530, "立": 2531, "站": 2532, "竟": 2533, "章": 2534, "童": 2535, "端": 2536, "競": 2537, "竹": 2538, "笆": 2539, "笈": 2540, "笏": 2541, "笑": 2542, "笙": 2543, "笛": 2544, "笠": 2545, "符": 2546, "笨": 2547, "笪": 2548, "第": 2549, "筆": 2550, "等": 2551, "筋": 2552, "筍": 2553, "筏": 2554, "筒": 2555, "答": 2556, "策": 2557, "筲": 2558, "筵": 2559, "筷": 2560, "箇": 2561, "箋": 2562, "箍": 2563, "箕": 2564, "算": 2565, "管": 2566, "箭": 2567, "箱": 2568, "箴": 2569, "節": 2570, "範": 2571, "篇": 2572, "築": 2573, "篋": 2574, "篙": 2575, "篤": 2576, "篳": 2577, "簁": 2578, "簍": 2579, "簡": 2580, "簪": 2581, "簫": 2582, "簽": 2583, "簾": 2584, "簿": 2585, "籃": 2586, "籌": 2587, "籍": 2588, "籐": 2589, "籠": 2590, "籤": 2591, "籬": 2592, "籮": 2593, "籲": 2594, "米": 2595, "籽": 2596, "粉": 2597, "粒": 2598, "粗": 2599, "粟": 2600, "粥": 2601, "粱": 2602, "粳": 2603, "粵": 2604, "粹": 2605, "粼": 2606, "粽": 2607, "精": 2608, "粿": 2609, "糉": 2610, "糊": 2611, "糍": 2612, "糕": 2613, "糖": 2614, "糞": 2615, "糟": 2616, "糧": 2617, "糬": 2618, "糯": 2619, "糰": 2620, "糴": 2621, "系": 2622, "糾": 2623, "紀": 2624, "約": 2625, "紅": 2626, "納": 2627, "紐": 2628, "紓": 2629, "純": 2630, "紗": 2631, "紙": 2632, "級": 2633, "紛": 2634, "素": 2635, "索": 2636, "紥": 2637, "紫": 2638, "紮": 2639, "累": 2640, "細": 2641, "紳": 2642, "紹": 2643, "終": 2644, "組": 2645, "結": 2646, "絕": 2647, "絞": 2648, "絡": 2649, "給": 2650, "絨": 2651, "統": 2652, "絲": 2653, "絶": 2654, "綁": 2655, "經": 2656, "綜": 2657, "綠": 2658, "綫": 2659, "維": 2660, "網": 2661, "綵": 2662, "綽": 2663, "綿": 2664, "緊": 2665, "緒": 2666, "緘": 2667, "線": 2668, "緣": 2669, "編": 2670, "緩": 2671, "緬": 2672, "緲": 2673, "練": 2674, "緻": 2675, "縉": 2676, "縊": 2677, "縛": 2678, "縣": 2679, "縫": 2680, "縮": 2681, "縱": 2682, "縷": 2683, "縹": 2684, "總": 2685, "績": 2686, "繁": 2687, "織": 2688, "繞": 2689, "繩": 2690, "繫": 2691, "繳": 2692, "繹": 2693, "繼": 2694, "續": 2695, "纏": 2696, "纔": 2697, "纖": 2698, "纜": 2699, "缸": 2700, "缺": 2701, "缽": 2702, "罄": 2703, "罅": 2704, "罐": 2705, "罔": 2706, "罕": 2707, "罟": 2708, "罩": 2709, "罪": 2710, "置": 2711, "罰": 2712, "署": 2713, "罵": 2714, "罷": 2715, "罹": 2716, "羅": 2717, "羈": 2718, "羊": 2719, "羌": 2720, "美": 2721, "羞": 2722, "羣": 2723, "群": 2724, "義": 2725, "羲": 2726, "羹": 2727, "羽": 2728, "翁": 2729, "翅": 2730, "翌": 2731, "習": 2732, "翔": 2733, "翕": 2734, "翠": 2735, "翡": 2736, "翩": 2737, "翰": 2738, "翱": 2739, "翹": 2740, "翻": 2741, "翼": 2742, "耀": 2743, "老": 2744, "考": 2745, "者": 2746, "而": 2747, "耍": 2748, "耐": 2749, "耕": 2750, "耗": 2751, "耘": 2752, "耳": 2753, "耶": 2754, "耷": 2755, "耿": 2756, "聆": 2757, "聊": 2758, "聖": 2759, "聘": 2760, "聚": 2761, "聞": 2762, "聯": 2763, "聰": 2764, "聲": 2765, "聳": 2766, "聶": 2767, "職": 2768, "聼": 2769, "聽": 2770, "聾": 2771, "肅": 2772, "肆": 2773, "肇": 2774, "肉": 2775, "肋": 2776, "肌": 2777, "肓": 2778, "肖": 2779, "肘": 2780, "肚": 2781, "肛": 2782, "肝": 2783, "股": 2784, "肥": 2785, "肨": 2786, "肩": 2787, "肯": 2788, "育": 2789, "肴": 2790, "肺": 2791, "胃": 2792, "背": 2793, "胎": 2794, "胚": 2795, "胡": 2796, "胭": 2797, "胸": 2798, "胺": 2799, "能": 2800, "脂": 2801, "脅": 2802, "脆": 2803, "脈": 2804, "脊": 2805, "脛": 2806, "脫": 2807, "脷": 2808, "脹": 2809, "脾": 2810, "腋": 2811, "腍": 2812, "腎": 2813, "腐": 2814, "腔": 2815, "腕": 2816, "腥": 2817, "腦": 2818, "腩": 2819, "腫": 2820, "腰": 2821, "腳": 2822, "腸": 2823, "腹": 2824, "腺": 2825, "腿": 2826, "膀": 2827, "膊": 2828, "膏": 2829, "膚": 2830, "膜": 2831, "膝": 2832, "膠": 2833, "膨": 2834, "膩": 2835, "膳": 2836, "膺": 2837, "膽": 2838, "臂": 2839, "臉": 2840, "臘": 2841, "臟": 2842, "臣": 2843, "臨": 2844, "自": 2845, "臭": 2846, "��": 2847, "致": 2848, "臺": 2849, "臻": 2850, "臼": 2851, "舂": 2852, "舅": 2853, "與": 2854, "興": 2855, "舉": 2856, "舊": 2857, "舌": 2858, "舍": 2859, "舐": 2860, "舒": 2861, "舔": 2862, "舖": 2863, "舞": 2864, "舟": 2865, "舢": 2866, "舨": 2867, "航": 2868, "般": 2869, "舶": 2870, "船": 2871, "艇": 2872, "艦": 2873, "良": 2874, "艱": 2875, "色": 2876, "艷": 2877, "艾": 2878, "芋": 2879, "芒": 2880, "芙": 2881, "芝": 2882, "芥": 2883, "芫": 2884, "芬": 2885, "芭": 2886, "芯": 2887, "花": 2888, "芳": 2889, "芹": 2890, "芽": 2891, "苑": 2892, "苔": 2893, "苗": 2894, "苟": 2895, "苣": 2896, "若": 2897, "苦": 2898, "英": 2899, "茂": 2900, "范": 2901, "茄": 2902, "茅": 2903, "茜": 2904, "茨": 2905, "茫": 2906, "茱": 2907, "茴": 2908, "茵": 2909, "茶": 2910, "茸": 2911, "茹": 2912, "荃": 2913, "荇": 2914, "草": 2915, "荊": 2916, "荒": 2917, "荔": 2918, "荷": 2919, "莆": 2920, "莉": 2921, "莊": 2922, "莎": 2923, "莓": 2924, "莞": 2925, "莫": 2926, "莽": 2927, "菁": 2928, "菇": 2929, "菊": 2930, "菌": 2931, "菓": 2932, "菜": 2933, "菠": 2934, "菩": 2935, "華": 2936, "菱": 2937, "菲": 2938, "菴": 2939, "萃": 2940, "萄": 2941, "萊": 2942, "萋": 2943, "萍": 2944, "萎": 2945, "萬": 2946, "萸": 2947, "萺": 2948, "落": 2949, "葉": 2950, "著": 2951, "葛": 2952, "葡": 2953, "董": 2954, "葫": 2955, "葬": 2956, "葳": 2957, "葵": 2958, "蒂": 2959, "蒐": 2960, "蒙": 2961, "蒜": 2962, "蒞": 2963, "蒡": 2964, "蒲": 2965, "蒸": 2966, "蒼": 2967, "蓀": 2968, "蓆": 2969, "蓉": 2970, "蓋": 2971, "蓓": 2972, "蓬": 2973, "蓮": 2974, "蓺": 2975, "蔓": 2976, "蔔": 2977, "蔗": 2978, "蔚": 2979, "蔥": 2980, "蔫": 2981, "蔬": 2982, "蔭": 2983, "蔽": 2984, "蕃": 2985, "蕉": 2986, "蕎": 2987, "蕙": 2988, "蕩": 2989, "蕪": 2990, "蕭": 2991, "蕾": 2992, "薄": 2993, "薇": 2994, "薈": 2995, "薏": 2996, "薑": 2997, "薩": 2998, "薪": 2999, "薯": 3000, "薰": 3001, "藉": 3002, "藍": 3003, "藏": 3004, "藐": 3005, "藕": 3006, "藝": 3007, "藤": 3008, "藥": 3009, "藹": 3010, "藻": 3011, "蘅": 3012, "蘆": 3013, "蘇": 3014, "蘋": 3015, "蘑": 3016, "蘭": 3017, "蘸": 3018, "蘿": 3019, "虎": 3020, "虐": 3021, "虓": 3022, "處": 3023, "虛": 3024, "虞": 3025, "號": 3026, "虧": 3027, "虱": 3028, "虹": 3029, "蚊": 3030, "蚌": 3031, "蚝": 3032, "蚵": 3033, "蚺": 3034, "蛇": 3035, "蛋": 3036, "蛙": 3037, "蛛": 3038, "蛟": 3039, "蛤": 3040, "蛾": 3041, "蜀": 3042, "蜂": 3043, "蜆": 3044, "蜇": 3045, "蜊": 3046, "蜘": 3047, "蜜": 3048, "蜢": 3049, "蝕": 3050, "蝗": 3051, "蝦": 3052, "蝨": 3053, "蝴": 3054, "蝶": 3055, "蝸": 3056, "融": 3057, "螞": 3058, "螢": 3059, "螺": 3060, "蟀": 3061, "蟆": 3062, "蟋": 3063, "蟠": 3064, "蟬": 3065, "蟲": 3066, "蟹": 3067, "蟻": 3068, "蠅": 3069, "蠔": 3070, "蠟": 3071, "蠢": 3072, "蠱": 3073, "蠶": 3074, "蠻": 3075, "血": 3076, "衆": 3077, "行": 3078, "衍": 3079, "術": 3080, "街": 3081, "衙": 3082, "衛": 3083, "衝": 3084, "衞": 3085, "衡": 3086, "衣": 3087, "表": 3088, "衫": 3089, "衰": 3090, "衲": 3091, "衷": 3092, "衾": 3093, "袁": 3094, "袋": 3095, "袖": 3096, "被": 3097, "裁": 3098, "裇": 3099, "裏": 3100, "裔": 3101, "裕": 3102, "裘": 3103, "裙": 3104, "補": 3105, "裝": 3106, "裡": 3107, "裳": 3108, "裴": 3109, "製": 3110, "複": 3111, "褒": 3112, "褦": 3113, "褪": 3114, "褲": 3115, "褸": 3116, "襟": 3117, "襪": 3118, "襯": 3119, "襲": 3120, "西": 3121, "要": 3122, "覆": 3123, "見": 3124, "規": 3125, "覓": 3126, "視": 3127, "親": 3128, "覲": 3129, "覺": 3130, "覽": 3131, "觀": 3132, "角": 3133, "解": 3134, "觴": 3135, "觸": 3136, "言": 3137, "訂": 3138, "計": 3139, "訊": 3140, "討": 3141, "訓": 3142, "訕": 3143, "託": 3144, "記": 3145, "訛": 3146, "訝": 3147, "訪": 3148, "設": 3149, "許": 3150, "訴": 3151, "診": 3152, "註": 3153, "証": 3154, "詆": 3155, "詐": 3156, "評": 3157, "詞": 3158, "詢": 3159, "試": 3160, "詩": 3161, "詬": 3162, "詭": 3163, "話": 3164, "該": 3165, "詳": 3166, "詹": 3167, "誅": 3168, "誇": 3169, "誌": 3170, "認": 3171, "誓": 3172, "誕": 3173, "誘": 3174, "語": 3175, "誠": 3176, "誡": 3177, "誤": 3178, "誨": 3179, "說": 3180, "説": 3181, "誰": 3182, "課": 3183, "誼": 3184, "調": 3185, "談": 3186, "請": 3187, "諒": 3188, "論": 3189, "諗": 3190, "諜": 3191, "諦": 3192, "諧": 3193, "諫": 3194, "諷": 3195, "諸": 3196, "諺": 3197, "諾": 3198, "謀": 3199, "謁": 3200, "謂": 3201, "謊": 3202, "謎": 3203, "謔": 3204, "謙": 3205, "講": 3206, "謝": 3207, "謢": 3208, "謬": 3209, "謹": 3210, "謾": 3211, "證": 3212, "譎": 3213, "譖": 3214, "識": 3215, "譚": 3216, "譜": 3217, "警": 3218, "譬": 3219, "譯": 3220, "議": 3221, "譴": 3222, "護": 3223, "譽": 3224, "讀": 3225, "變": 3226, "讎": 3227, "讓": 3228, "讚": 3229, "谷": 3230, "豁": 3231, "豂": 3232, "豆": 3233, "豈": 3234, "豉": 3235, "豎": 3236, "豐": 3237, "豚": 3238, "象": 3239, "豪": 3240, "豫": 3241, "豬": 3242, "豹": 3243, "貂": 3244, "貌": 3245, "貓": 3246, "貝": 3247, "負": 3248, "財": 3249, "貢": 3250, "貧": 3251, "貨": 3252, "販": 3253, "貪": 3254, "貫": 3255, "責": 3256, "貴": 3257, "貶": 3258, "買": 3259, "貸": 3260, "費": 3261, "貼": 3262, "貿": 3263, "賀": 3264, "賃": 3265, "資": 3266, "賈": 3267, "賊": 3268, "賒": 3269, "賓": 3270, "賜": 3271, "賞": 3272, "賢": 3273, "賣": 3274, "賤": 3275, "賦": 3276, "質": 3277, "賬": 3278, "賭": 3279, "賴": 3280, "賺": 3281, "購": 3282, "賽": 3283, "贅": 3284, "贈": 3285, "贊": 3286, "贏": 3287, "贛": 3288, "贼": 3289, "赤": 3290, "赫": 3291, "走": 3292, "赴": 3293, "起": 3294, "趁": 3295, "超": 3296, "越": 3297, "趌": 3298, "趕": 3299, "趙": 3300, "趣": 3301, "趨": 3302, "足": 3303, "趴": 3304, "趷": 3305, "趺": 3306, "趾": 3307, "跋": 3308, "跌": 3309, "跑": 3310, "跛": 3311, "距": 3312, "跟": 3313, "跡": 3314, "跣": 3315, "跨": 3316, "跪": 3317, "路": 3318, "跳": 3319, "踎": 3320, "踏": 3321, "踐": 3322, "踢": 3323, "踩": 3324, "踪": 3325, "踱": 3326, "踹": 3327, "蹄": 3328, "蹆": 3329, "蹈": 3330, "蹋": 3331, "蹟": 3332, "蹤": 3333, "蹲": 3334, "蹺": 3335, "躁": 3336, "躇": 3337, "躉": 3338, "躊": 3339, "躍": 3340, "躝": 3341, "身": 3342, "躬": 3343, "躲": 3344, "車": 3345, "軌": 3346, "軍": 3347, "軒": 3348, "軚": 3349, "軟": 3350, "較": 3351, "載": 3352, "輊": 3353, "輋": 3354, "輔": 3355, "輕": 3356, "輘": 3357, "輛": 3358, "輝": 3359, "輟": 3360, "輩": 3361, "輪": 3362, "輯": 3363, "輷": 3364, "輸": 3365, "輻": 3366, "輾": 3367, "轄": 3368, "轆": 3369, "轉": 3370, "轍": 3371, "轎": 3372, "轔": 3373, "轡": 3374, "辛": 3375, "辜": 3376, "辣": 3377, "辦": 3378, "辨": 3379, "辭": 3380, "辯": 3381, "辰": 3382, "辱": 3383, "農": 3384, "迂": 3385, "迅": 3386, "迍": 3387, "迎": 3388, "近": 3389, "返": 3390, "迦": 3391, "迪": 3392, "迫": 3393, "述": 3394, "迴": 3395, "迷": 3396, "追": 3397, "迾": 3398, "退": 3399, "送": 3400, "逃": 3401, "逆": 3402, "透": 3403, "逐": 3404, "途": 3405, "逗": 3406, "這": 3407, "通": 3408, "逝": 3409, "逞": 3410, "速": 3411, "造": 3412, "逢": 3413, "連": 3414, "週": 3415, "進": 3416, "逸": 3417, "逹": 3418, "逼": 3419, "逾": 3420, "遁": 3421, "遂": 3422, "遇": 3423, "遊": 3424, "運": 3425, "遍": 3426, "過": 3427, "遏": 3428, "道": 3429, "達": 3430, "違": 3431, "遙": 3432, "遜": 3433, "遞": 3434, "遠": 3435, "遢": 3436, "遣": 3437, "適": 3438, "遭": 3439, "遮": 3440, "遲": 3441, "遴": 3442, "遵": 3443, "遷": 3444, "選": 3445, "遺": 3446, "遼": 3447, "避": 3448, "邀": 3449, "還": 3450, "邈": 3451, "邊": 3452, "邋": 3453, "邏": 3454, "那": 3455, "邦": 3456, "邨": 3457, "邪": 3458, "邰": 3459, "邵": 3460, "邸": 3461, "郁": 3462, "郊": 3463, "郎": 3464, "郝": 3465, "部": 3466, "郭": 3467, "郵": 3468, "都": 3469, "鄂": 3470, "鄉": 3471, "鄙": 3472, "鄧": 3473, "鄭": 3474, "鄰": 3475, "酋": 3476, "酌": 3477, "配": 3478, "酒": 3479, "酣": 3480, "酥": 3481, "酪": 3482, "酬": 3483, "酮": 3484, "酱": 3485, "酷": 3486, "酸": 3487, "醇": 3488, "醉": 3489, "醋": 3490, "醒": 3491, "醜": 3492, "醫": 3493, "醬": 3494, "醺": 3495, "釀": 3496, "采": 3497, "釋": 3498, "里": 3499, "重": 3500, "野": 3501, "量": 3502, "金": 3503, "釗": 3504, "釘": 3505, "釜": 3506, "針": 3507, "釣": 3508, "釵": 3509, "鈍": 3510, "鈔": 3511, "鈕": 3512, "鈞": 3513, "鈴": 3514, "鈿": 3515, "鉗": 3516, "鉛": 3517, "鉤": 3518, "鉸": 3519, "銀": 3520, "銃": 3521, "銅": 3522, "銘": 3523, "銳": 3524, "銷": 3525, "鋁": 3526, "鋒": 3527, "鋤": 3528, "鋪": 3529, "鋼": 3530, "錄": 3531, "錐": 3532, "錘": 3533, "錢": 3534, "錦": 3535, "錫": 3536, "錯": 3537, "錶": 3538, "鍊": 3539, "鍋": 3540, "鍚": 3541, "鍵": 3542, "鍾": 3543, "鎖": 3544, "鎗": 3545, "鎭": 3546, "鎮": 3547, "鏈": 3548, "鏟": 3549, "鏡": 3550, "鏢": 3551, "鏰": 3552, "鐘": 3553, "鐡": 3554, "鐵": 3555, "鐸": 3556, "鑄": 3557, "鑊": 3558, "鑑": 3559, "鑫": 3560, "鑲": 3561, "鑼": 3562, "鑽": 3563, "鑿": 3564, "長": 3565, "門": 3566, "閂": 3567, "閃": 3568, "閉": 3569, "開": 3570, "閏": 3571, "閑": 3572, "閒": 3573, "間": 3574, "閘": 3575, "閣": 3576, "閨": 3577, "閩": 3578, "閪": 3579, "閱": 3580, "閻": 3581, "闆": 3582, "闊": 3583, "闌": 3584, "闔": 3585, "闕": 3586, "闖": 3587, "關": 3588, "闢": 3589, "阜": 3590, "阪": 3591, "阱": 3592, "防": 3593, "阻": 3594, "阿": 3595, "陀": 3596, "陂": 3597, "附": 3598, "陋": 3599, "陌": 3600, "降": 3601, "限": 3602, "陞": 3603, "院": 3604, "陣": 3605, "除": 3606, "陪": 3607, "陰": 3608, "陳": 3609, "陵": 3610, "陶": 3611, "陷": 3612, "陸": 3613, "陽": 3614, "隅": 3615, "隆": 3616, "隊": 3617, "階": 3618, "隔": 3619, "隙": 3620, "際": 3621, "障": 3622, "隧": 3623, "隨": 3624, "險": 3625, "隱": 3626, "隴": 3627, "隸": 3628, "隻": 3629, "雀": 3630, "雁": 3631, "雄": 3632, "雅": 3633, "集": 3634, "雋": 3635, "雌": 3636, "雍": 3637, "雕": 3638, "雖": 3639, "雙": 3640, "雜": 3641, "雞": 3642, "離": 3643, "難": 3644, "雨": 3645, "雪": 3646, "雲": 3647, "零": 3648, "雷": 3649, "電": 3650, "需": 3651, "霄": 3652, "震": 3653, "霉": 3654, "霎": 3655, "霓": 3656, "霖": 3657, "霜": 3658, "霧": 3659, "露": 3660, "霸": 3661, "霾": 3662, "靈": 3663, "青": 3664, "靖": 3665, "靚": 3666, "靜": 3667, "非": 3668, "靠": 3669, "面": 3670, "革": 3671, "靴": 3672, "靶": 3673, "鞋": 3674, "鞍": 3675, "鞦": 3676, "鞭": 3677, "韆": 3678, "韌": 3679, "韓": 3680, "韭": 3681, "音": 3682, "韻": 3683, "響": 3684, "頁": 3685, "頂": 3686, "項": 3687, "順": 3688, "須": 3689, "頌": 3690, "預": 3691, "頑": 3692, "頒": 3693, "頓": 3694, "頗": 3695, "領": 3696, "頤": 3697, "頭": 3698, "頸": 3699, "頹": 3700, "頻": 3701, "題": 3702, "額": 3703, "顏": 3704, "顔": 3705, "願": 3706, "顛": 3707, "類": 3708, "顧": 3709, "顯": 3710, "顱": 3711, "風": 3712, "颱": 3713, "飄": 3714, "飛": 3715, "食": 3716, "飢": 3717, "飩": 3718, "飯": 3719, "飲": 3720, "飼": 3721, "飽": 3722, "飾": 3723, "餃": 3724, "餅": 3725, "餉": 3726, "養": 3727, "餋": 3728, "餐": 3729, "餒": 3730, "餓": 3731, "餘": 3732, "餛": 3733, "館": 3734, "餮": 3735, "餵": 3736, "餸": 3737, "餼": 3738, "餾": 3739, "饅": 3740, "饌": 3741, "饑": 3742, "饒": 3743, "饕": 3744, "首": 3745, "香": 3746, "馨": 3747, "馩": 3748, "馬": 3749, "馭": 3750, "馮": 3751, "馳": 3752, "駁": 3753, "駐": 3754, "駒": 3755, "駕": 3756, "駛": 3757, "駝": 3758, "駟": 3759, "駱": 3760, "駿": 3761, "騅": 3762, "騎": 3763, "騙": 3764, "騭": 3765, "騮": 3766, "騰": 3767, "騷": 3768, "騾": 3769, "驀": 3770, "驅": 3771, "驕": 3772, "驗": 3773, "驚": 3774, "驟": 3775, "驥": 3776, "驪": 3777, "骨": 3778, "骯": 3779, "骰": 3780, "骸": 3781, "骹": 3782, "髀": 3783, "髒": 3784, "髓": 3785, "體": 3786, "高": 3787, "髮": 3788, "髻": 3789, "鬆": 3790, "鬚": 3791, "鬠": 3792, "鬢": 3793, "鬥": 3794, "鬧": 3795, "鬱": 3796, "鬼": 3797, "魁": 3798, "魂": 3799, "魄": 3800, "魅": 3801, "魏": 3802, "魔": 3803, "魚": 3804, "魯": 3805, "魷": 3806, "鮑": 3807, "鮟": 3808, "鮫": 3809, "鮭": 3810, "鮮": 3811, "鯇": 3812, "鯉": 3813, "鯊": 3814, "鯖": 3815, "鯛": 3816, "鯪": 3817, "鰂": 3818, "鰭": 3819, "鰲": 3820, "鰻": 3821, "鱇": 3822, "鱈": 3823, "鱔": 3824, "鱗": 3825, "鱲": 3826, "鱷": 3827, "鱸": 3828, "鲁": 3829, "鳥": 3830, "鳩": 3831, "鳳": 3832, "鳴": 3833, "鳶": 3834, "鴉": 3835, "鴛": 3836, "鴦": 3837, "鴨": 3838, "鴻": 3839, "鴿": 3840, "鵝": 3841, "鵡": 3842, "鵪": 3843, "鵬": 3844, "鵲": 3845, "鶉": 3846, "鶴": 3847, "鷄": 3848, "鷯": 3849, "鷹": 3850, "鸚": 3851, "鸝": 3852, "鸞": 3853, "鹅": 3854, "鹹": 3855, "鹼": 3856, "鹽": 3857, "鹿": 3858, "麒": 3859, "麗": 3860, "麝": 3861, "麟": 3862, "麥": 3863, "麪": 3864, "麵": 3865, "麻": 3866, "麼": 3867, "黃": 3868, "黎": 3869, "黏": 3870, "黐": 3871, "黑": 3872, "默": 3873, "黚": 3874, "黛": 3875, "黜": 3876, "點": 3877, "黨": 3878, "黯": 3879, "鼆": 3880, "鼎": 3881, "鼓": 3882, "鼙": 3883, "鼠": 3884, "鼻": 3885, "齊": 3886, "齋": 3887, "齒": 3888, "齡": 3889, "齣": 3890, "齪": 3891, "齷": 3892, "龍": 3893, "龐": 3894, "龜": 3895, "龢": 3896, "更": 3897, "來": 3898, "不": 3899, "年": 3900, "聯": 3901, "料": 3902, "利": 3903, "立": 3904, "行": 3905, "︰": 3906, "﹔": 3907, "﹖": 3908, "！": 3909, "（": 3910, "）": 3911, "，": 3912, "－": 3913, "．": 3914, "／": 3915, "：": 3916, "；": 3917, "？": 3918, "ａ": 3919, "ｂ": 3920, "～": 3921, "": 3922, "|": 0, "[UNK]": 3923, "[PAD]": 3924}