Upload folder using huggingface_hub

.github/workflows/pull_format.yml

@@ -0,0 +1,43 @@
+name: pull format
+
+on: [pull_request]
+
+permissions:
+  contents: write
+
+jobs:
+  pull_format:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+      fail-fast: false
+
+    continue-on-error: true
+
+    steps:
+      - name: checkout
+        continue-on-error: true
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Black
+        run: pip install "black[jupyter]"
+
+      - name: Run Black
+        # run: black $(git ls-files '*.py')
+        run: black .
+
+      - name: Commit Back
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Apply Code Formatter Change

.github/workflows/push_format.yml

@@ -0,0 +1,57 @@
+name: push format
+
+on:
+  push:
+    branches:
+      - master
+      - dev
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  push_format:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{github.ref_name}}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Black
+        run: pip install "black[jupyter]"
+
+      - name: Run Black
+        # run: black $(git ls-files '*.py')
+        run: black .
+
+      - name: Commit Back
+        continue-on-error: true
+        id: commitback
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add --all
+          git commit -m "Format code"
+
+      - name: Create Pull Request
+        if: steps.commitback.outcome == 'success'
+        continue-on-error: true
+        uses: peter-evans/create-pull-request@v5
+        with:
+          delete-branch: true
+          body: Apply Code Formatter Change
+          title: Apply Code Formatter Change
+          commit-message: Automatic code format

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.8
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+
+  - repo: https://github.com/psf/black
+    rev: 23.12.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        files: ^.*\.(py|md|rst|yml)$
+        args: [-L=fro]

Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.10.12
+
+# Install Git LFS
+RUN apt-get update -y
+RUN apt-get install -y git-lfs
+RUN git lfs install
+
+# Switch to User
+RUN useradd -m -u 1000 user
+USER user
+
+# Setup Environment Variables
+ENV HOME=/home/user
+ENV PATH=/home/user/.local/bin:$PATH
+
+# Init Working Directory
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+
+# Install Python Requirements
+RUN pip install --no-cache-dir --upgrade pip
+RUN pip install -r requirements.txt
+
+# Download Bert, CLAP and WavLM Models
+RUN rm -rf bert/deberta-v2-large-japanese-char-wwm
+RUN git clone https://huggingface.co/ku-nlp/deberta-v2-large-japanese-char-wwm bert/deberta-v2-large-japanese-char-wwm --depth=1
+RUN cd bert/deberta-v2-large-japanese-char-wwm && git lfs pull
+RUN rm -rf emotional/clap-htsat-fused
+RUN git clone https://huggingface.co/laion/clap-htsat-fused emotional/clap-htsat-fused --depth=1
+RUN cd emotional/clap-htsat-fused && git lfs pull
+RUN rm -rf slm/wavlm-base-plus
+RUN git clone https://huggingface.co/microsoft/wavlm-base-plus slm/wavlm-base-plus --depth=1
+RUN cd slm/wavlm-base-plus && git lfs pull
+
+# Start Web Server
+ENV GRADIO_SERVER_NAME=0.0.0.0
+CMD ["python", "webui.py"]

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

README.md

@@ -1,10 +1,51 @@
 ---
-title: Bert VITS2 YNS
-emoji: 👀
-colorFrom: pink
-colorTo: indigo
+title: AI Yosuga No Sora
+emoji: 🐰
+colorFrom: gray
+colorTo: blue
 sdk: docker
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<div align="center">
+
+<img alt="LOGO" src="https://cdn.jsdelivr.net/gh/fishaudio/fish-diffusion@main/images/logo_512x512.png" width="256" height="256" />
+
+# Bert-VITS2
+
+VITS2 Backbone with multilingual bert
+
+For quick guide, please refer to `webui_preprocess.py`.
+
+简易教程请参见 `webui_preprocess.py`。
+
+## 请注意，本项目核心思路来源于[anyvoiceai/MassTTS](https://github.com/anyvoiceai/MassTTS) 一个非常好的tts项目
+## MassTTS的演示demo为[ai版峰哥锐评峰哥本人,并找回了在金三角失落的腰子](https://www.bilibili.com/video/BV1w24y1c7z9)
+
+[//]: # (## 本项目与[PlayVoice/vits_chinese]&#40;https://github.com/PlayVoice/vits_chinese&#41; 没有任何关系)
+
+[//]: # ()
+[//]: # (本仓库来源于之前朋友分享了ai峰哥的视频，本人被其中的效果惊艳，在自己尝试MassTTS以后发现fs在音质方面与vits有一定差距，并且training的pipeline比vits更复杂，因此按照其思路将bert)
+
+## 成熟的旅行者/开拓者/舰长/博士/sensei/猎魔人/喵喵露/V应当参阅代码自己学习如何训练。
+
+### 严禁将此项目用于一切违反《中华人民共和国宪法》，《中华人民共和国刑法》，《中华人民共和国治安管理处罚法》和《中华人民共和国民法典》之用途。
+### 严禁用于任何政治相关用途。
+#### Video:https://www.bilibili.com/video/BV1hp4y1K78E
+#### Demo:https://www.bilibili.com/video/BV1TF411k78w
+#### QQ Group：815818430
+## References
++ [anyvoiceai/MassTTS](https://github.com/anyvoiceai/MassTTS)
++ [jaywalnut310/vits](https://github.com/jaywalnut310/vits)
++ [p0p4k/vits2_pytorch](https://github.com/p0p4k/vits2_pytorch)
++ [svc-develop-team/so-vits-svc](https://github.com/svc-develop-team/so-vits-svc)
++ [PaddlePaddle/PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech)
++ [emotional-vits](https://github.com/innnky/emotional-vits)
++ [fish-speech](https://github.com/fishaudio/fish-speech)
++ [Bert-VITS2-UI](https://github.com/jiangyuxiaoxiao/Bert-VITS2-UI)
+## 感谢所有贡献者作出的努力
+<a href="https://github.com/fishaudio/Bert-VITS2/graphs/contributors" target="_blank">
+  <img src="https://contrib.rocks/image?repo=fishaudio/Bert-VITS2"/>
+</a>
+
+[//]: # (# 本项目所有代码引用均已写明，bert部分代码思路来源于[AI峰哥]&#40;https://www.bilibili.com/video/BV1w24y1c7z9&#41;，与[vits_chinese]&#40;https://github.com/PlayVoice/vits_chinese&#41;无任何关系。欢迎各位查阅代码。同时，我们也对该开发者的[碰瓷，乃至开盒开发者的行为]&#40;https://www.bilibili.com/read/cv27101514/&#41;表示强烈谴责。)

attentions.py

@@ -0,0 +1,464 @@
+import math
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import commons
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
+
+
+class Encoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        window_size=4,
+        isflow=True,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        # if isflow:
+        #  cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1)
+        #  self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1)
+        #  self.cond_layer = weight_norm(cond_layer, name='weight')
+        #  self.gin_channels = 256
+        self.cond_layer_idx = self.n_layers
+        if "gin_channels" in kwargs:
+            self.gin_channels = kwargs["gin_channels"]
+            if self.gin_channels != 0:
+                self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels)
+                # vits2 says 3rd block, so idx is 2 by default
+                self.cond_layer_idx = (
+                    kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
+                )
+                logging.debug(self.gin_channels, self.cond_layer_idx)
+                assert (
+                    self.cond_layer_idx < self.n_layers
+                ), "cond_layer_idx should be less than n_layers"
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    window_size=window_size,
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, g=None):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            if i == self.cond_layer_idx and g is not None:
+                g = self.spk_emb_linear(g.transpose(1, 2))
+                g = g.transpose(1, 2)
+                x = x + g
+                x = x * x_mask
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class Decoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        proximal_bias=False,
+        proximal_init=True,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    proximal_bias=proximal_bias,
+                    proximal_init=proximal_init,
+                )
+            )
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                    causal=True,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
+        x: decoder input
+        h: encoder output
+        """
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
+            device=x.device, dtype=x.dtype
+        )
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(
+        self,
+        channels,
+        out_channels,
+        n_heads,
+        p_dropout=0.0,
+        window_size=None,
+        heads_share=True,
+        block_length=None,
+        proximal_bias=False,
+        proximal_init=False,
+    ):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+            self.emb_rel_v = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+        if self.window_size is not None:
+            assert (
+                t_s == t_t
+            ), "Relative attention is only available for self-attention."
+            key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(
+                query / math.sqrt(self.k_channels), key_relative_embeddings
+            )
+            scores_local = self._relative_position_to_absolute_position(rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            assert t_s == t_t, "Proximal bias is only available for self-attention."
+            scores = scores + self._attention_bias_proximal(t_s).to(
+                device=scores.device, dtype=scores.dtype
+            )
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                assert (
+                    t_s == t_t
+                ), "Local attention is only available for self-attention."
+                block_mask = (
+                    torch.ones_like(scores)
+                    .triu(-self.block_length)
+                    .tril(self.block_length)
+                )
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_v, t_s
+            )
+            output = output + self._matmul_with_relative_values(
+                relative_weights, value_relative_embeddings
+            )
+        output = (
+            output.transpose(2, 3).contiguous().view(b, d, t_t)
+        )  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
+        x: [b, h, l, m]
+        y: [h or 1, m, d]
+        ret: [b, h, l, d]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
+
+    def _matmul_with_relative_keys(self, x, y):
+        """
+        x: [b, h, l, d]
+        y: [h or 1, m, d]
+        ret: [b, h, l, m]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
+            )
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[
+            :, slice_start_position:slice_end_position
+        ]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
+        x: [b, h, l, 2*l-1]
+        ret: [b, h, l, l]
+        """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
+        )
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
+            :, :, :length, length - 1 :
+        ]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
+        x: [b, h, l, l]
+        ret: [b, h, l, 2*l-1]
+        """
+        batch, heads, length, _ = x.size()
+        # pad along column
+        x = F.pad(
+            x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
+        )
+        x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
+        Args:
+          length: an integer scalar.
+        Returns:
+          a Tensor with shape [1, 1, length, length]
+        """
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        filter_channels,
+        kernel_size,
+        p_dropout=0.0,
+        activation=None,
+        causal=False,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

bert/Erlangshen-DeBERTa-v2-710M-Chinese/config.json

@@ -0,0 +1,35 @@
+{
+    "model_type": "deberta-v2",
+    "architectures": [
+        "DebertaV2ForMaskedLM"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "attention_head_size": 64,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "max_position_embeddings": 512,
+    "relative_attention": true,
+    "position_buckets": 256,
+    "norm_rel_ebd": "layer_norm",
+    "share_att_key": true,
+    "pos_att_type": [
+        "p2c",
+        "c2p"
+    ],
+    "conv_kernel_size": 3,
+    "pooler_dropout": 0,
+    "pooler_hidden_act": "gelu",
+    "pooler_hidden_size": 1536,
+    "conv_act": "gelu",
+    "layer_norm_eps": 1e-7,
+    "max_relative_positions": -1,
+    "position_biased_input": false,
+    "num_attention_heads": 24,
+    "num_hidden_layers": 24,
+    "type_vocab_size": 0,
+    "num_labels": 119,
+    "vocab_size": 12800
+}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+    "do_lower_case": true,
+    "do_basic_tokenize": true,
+    "never_split": null,
+    "unk_token": "[UNK]",
+    "sep_token": "[SEP]",
+    "pad_token": "[PAD]",
+    "cls_token": "[CLS]",
+    "mask_token": "[MASK]",
+    "tokenize_chinese_chars": true,
+    "strip_accents": null,
+    "special_tokens_map_file": null,
+    "name_or_path": "/cognitive_comp/gaoxinyu/pretrained_model/bert-1.3B",
+    "tokenizer_class": "BertTokenizer"
+}

bert/Erlangshen-DeBERTa-v2-710M-Chinese/vocab.txt

@@ -0,0 +1,12800 @@
+[PAD]
+[CLS]
+[SEP]
+[UNK]
+[MASK]
+[unused1]
+[unused2]
+[unused3]
+[unused4]
+[unused5]
+[unused6]
+[unused7]
+[unused8]
+[unused9]
+[unused10]
+[unused11]
+[unused12]
+[unused13]
+[unused14]
+[unused15]
+[unused16]
+[unused17]
+[unused18]
+[unused19]
+[unused20]
+[unused21]
+[unused22]
+[unused23]
+[unused24]
+[unused25]
+[unused26]
+[unused27]
+[unused28]
+[unused29]
+[unused30]
+[unused31]
+[unused32]
+[unused33]
+[unused34]
+[unused35]
+[unused36]
+[unused37]
+[unused38]
+[unused39]
+[unused40]
+[unused41]
+[unused42]
+[unused43]
+[unused44]
+[unused45]
+[unused46]
+[unused47]
+[unused48]
+[unused49]
+[unused50]
+[unused51]
+[unused52]
+[unused53]
+[unused54]
+[unused55]
+[unused56]
+[unused57]
+[unused58]
+[unused59]
+[unused60]
+[unused61]
+[unused62]
+[unused63]
+[unused64]
+[unused65]
+[unused66]
+[unused67]
+[unused68]
+[unused69]
+[unused70]
+[unused71]
+[unused72]
+[unused73]
+[unused74]
+[unused75]
+[unused76]
+[unused77]
+[unused78]
+[unused79]
+[unused80]
+[unused81]
+[unused82]
+[unused83]
+[unused84]
+[unused85]
+[unused86]
+[unused87]
+[unused88]
+[unused89]
+[unused90]
+[unused91]
+[unused92]
+[unused93]
+[unused94]
+[unused95]
+[unused96]
+[unused97]
+[unused98]
+[unused99]
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+^
+_
+`
+{
+|
+}
+~
+·
+–
+—
+‘
+’
+‛
+“
+”
+„
+‟
+…
+‧
+、
+。
+〃
+〈
+〉
+《
+》
+「
+」
+『
+』
+【
+】
+〔
+〕
+〖
+〗
+〜
+〝
+〞
+〰
+﹏
+﹑
+﹔
+！
+＂
+＃
+＄
+％
+＆
+＇
+（
+）
+＊
+＋
+，
+－
+：
+；
+＜
+＝
+＞
+？
+＠
+［
+＼
+］
+＾
+＿
+｀
+｛
+｜
+｝
+～
+｡
+｢
+｣
+､
+0
+##0
+1
+##1
+2
+##2
+3
+##3
+4
+##4
+5
+##5
+6
+##6
+7
+##7
+8
+##8
+9
+##9
+一
+丁
+七
+丄
+丅
+丆
+万
+丈
+三
+上
+下
+丌
+不
+与
+丏
+丐
+丑
+专
+且
+丕
+世
+丘
+丙
+业
+丛
+东
+丝
+丞
+両
+丢
+两
+严
+丧
+丨
+个
+丫
+中
+丰
+串
+临
+丶
+丸
+丹
+为
+主
+丼
+丽
+举
+丿
+乂
+乃
+乄
+久
+乇
+么
+义
+之
+乌
+乍
+乎
+乏
+乐
+乒
+乓
+乔
+乖
+乗
+乘
+乙
+乛
+乜
+九
+乞
+也
+习
+乡
+书
+乩
+买
+乱
+乳
+乸
+乾
+亀
+了
+亇
+予
+争
+亊
+事
+二
+亍
+于
+亏
+云
+互
+亓
+五
+井
+亘
+亚
+些
+亜
+亟
+亡
+亢
+交
+亥
+亦
+产
+亨
+亩
+享
+京
+亭
+亮
+亲
+亳
+亵
+亶
+亸
+亹
+人
+亻
+亽
+亾
+亿
+什
+仁
+仂
+仃
+仄
+仅
+仆
+仇
+仉
+今
+介
+仍
+从
+仏
+仑
+仓
+仔
+仕
+他
+仗
+付
+仙
+仝
+仞
+仟
+仡
+代
+令
+以
+仨
+仩
+仪
+仫
+们
+仮
+仰
+仲
+仳
+仵
+件
+价
+任
+仼
+份
+仿
+企
+伉
+伊
+伋
+伍
+伎
+伏
+伐
+休
+众
+优
+伙
+会
+伛
+伝
+伞
+伟
+传
+伢
+伤
+伥
+伦
+伧
+伪
+伫
+伯
+估
+伱
+伲
+伴
+伶
+伷
+伸
+伺
+似
+伽
+伾
+佀
+佃
+但
+佉
+位
+低
+住
+佐
+佑
+体
+何
+佗
+佘
+余
+佚
+佛
+作
+佝
+佞
+佟
+你
+佢
+佣
+佤
+佥
+佧
+佩
+佬
+佯
+佰
+佳
+佴
+佶
+佷
+佺
+佻
+佼
+佾
+使
+侁
+侂
+侃
+侄
+來
+侈
+侉
+例
+侍
+侏
+侑
+侔
+侗
+侘
+供
+依
+侠
+価
+侣
+侥
+侦
+侧
+侨
+侩
+侪
+侬
+侮
+侯
+侵
+便
+促
+俄
+俅
+俊
+俎
+俏
+俐
+俑
+俗
+俘
+俚
+俛
+俜
+保
+俞
+俟
+信
+俢
+俣
+俤
+俦
+俨
+俩
+俪
+俬
+俭
+修
+俯
+俱
+俳
+俶
+俸
+俺
+俾
+倅
+倌
+倍
+倏
+倒
+倓
+倔
+倕
+倘
+候
+倚
+倜
+倞
+借
+倡
+値
+倥
+倦
+倧
+倨
+倩
+倪
+倬
+倭
+倮
+倶
+债
+倻
+值
+倾
+偁
+偃
+假
+偈
+偌
+偍
+偎
+偏
+偓
+偕
+做
+停
+偠
+健
+偬
+偰
+偲
+偶
+偷
+偻
+偾
+偿
+傀
+傅
+傈
+傉
+傍
+傒
+傕
+傣
+傥
+傧
+储
+傩
+催
+傲
+傺
+傻
+働
+像
+僖
+僚
+僜
+僣
+僦
+僧
+僬
+僭
+僮
+僰
+僳
+僵
+僻
+僾
+儁
+儆
+儇
+儋
+儒
+儞
+儡
+儿
+兀
+允
+元
+兄
+充
+兆
+先
+光
+克
+免
+兎
+児
+兑
+兔
+兕
+兖
+党
+兜
+兢
+入
+全
+八
+公
+六
+兮
+兰
+共
+兲
+关
+兴
+兵
+其
+具
+典
+兹
+养
+兼
+兽
+冀
+内
+円
+冇
+冈
+冉
+册
+再
+冏
+冒
+冕
+冗
+写
+冚
+军
+农
+冠
+冢
+冤
+冥
+冧
+冨
+冬
+冮
+冯
+冰
+冲
+决
+冴
+况
+冶
+冷
+冻
+冼
+冽
+净
+凃
+凄
+准
+凇
+凉
+凊
+凋
+凌
+减
+凑
+凖
+凛
+凝
+几
+凡
+凤
+処
+凪
+凫
+凭
+凯
+凰
+凳
+凶
+凸
+凹
+出
+击
+凼
+函
+凿
+刀
+刁
+刂
+刃
+分
+切
+刈
+刊
+刋
+刍
+刎
+刑
+划
+刖
+列
+刘
+则
+刚
+创
+初
+删
+判
+刨
+利
+别
+刬
+刭
+刮
+到
+刳
+制
+刷
+券
+刹
+刺
+刻
+刽
+刿
+剀
+剁
+剂
+剃
+剅
+削
+剌
+前
+剐
+剑
+剔
+剖
+剜
+剞
+剡
+剣
+剥
+剧
+剩
+剪
+副
+割
+剽
+剿
+劂
+劄
+劈
+劏
+劓
+力
+劝
+办
+功
+加
+务
+劢
+劣
+动
+助
+努
+劫
+劬
+劭
+励
+劲
+劳
+劵
+効
+劻
+劼
+劾
+势
+勃
+勅
+勇
+勉
+勋
+勍
+勐
+勑
+勒
+勔
+勖
+勘
+募
+勠
+勤
+勰
+勲
+勳
+勷
+勺
+勾
+勿
+匀
+匂
+匄
+包
+匆
+匈
+匋
+匍
+匏
+匐
+匕
+化
+北
+匙
+匚
+匜
+匝
+匠
+匡
+匣
+匦
+匪
+匮
+匹
+区
+医
+匾
+匿
+區
+十
+千
+卅
+升
+午
+卉
+半
+卌
+卍
+华
+协
+卐
+卑
+卒
+卓
+单
+卖
+南
+単
+博
+卜
+卝
+卞
+卟
+占
+卡
+卢
+卣
+卤
+卦
+卧
+卨
+卫
+卬
+卮
+卯
+印
+危
+卲
+即
+却
+卵
+卷
+卸
+卺
+卽
+卿
+厂
+厄
+厅
+历
+厉
+压
+厌
+厍
+厓
+厔
+厕
+厘
+厚
+厝
+原
+厢
+厣
+厥
+厦
+厨
+厩
+厮
+厶
+去
+县
+叁
+参
+叆
+又
+叉
+及
+友
+双
+反
+収
+发
+叒
+叔
+叕
+取
+受
+变
+叙
+叛
+叟
+叠
+叡
+口
+古
+句
+另
+叨
+叩
+只
+叫
+召
+叭
+叮
+可
+台
+叱
+史
+右
+叵
+叶
+号
+司
+叹
+叻
+叼
+叽
+吁
+吃
+各
+吅
+吆
+吇
+合
+吉
+吊
+吋
+同
+名
+后
+吏
+吐
+向
+吒
+吓
+吔
+吕
+吖
+吗
+吙
+吚
+君
+吝
+吞
+吟
+吠
+吡
+吥
+否
+吧
+吨
+吩
+含
+听
+吭
+吮
+启
+吱
+吲
+吴
+吵
+吸
+吹
+吻
+吼
+吽
+吾
+吿
+呀
+呃
+呆
+呈
+呉
+告
+呋
+呎
+呐
+呑
+呒
+呓
+呔
+呕
+呖
+呗
+员
+呙
+呛
+呜
+呢
+呣
+呤
+呦
+周
+呪
+呬
+呯
+呱
+呲
+味
+呴
+呵
+呶
+呷
+呸
+呻
+呼
+命
+呾
+咀
+咁
+咂
+咄
+咆
+咋
+和
+咎
+咏
+咐
+咒
+咔
+咕
+咖
+咗
+咘
+咙
+咚
+咛
+咝
+咢
+咣
+咤
+咥
+咦
+咧
+咨
+咩
+咪
+咫
+咬
+咭
+咯
+咱
+咲
+咳
+咴
+咸
+咻
+咽
+咾
+咿
+哀
+品
+哂
+哄
+哆
+哇
+哈
+哉
+哋
+哌
+响
+哎
+哏
+哐
+哑
+哒
+哓
+哔
+哕
+哗
+哙
+哚
+哝
+哞
+哟
+哥
+哦
+哧
+哨
+哩
+哪
+哭
+哮
+哲
+哺
+哼
+哽
+哿
+唁
+唃
+唆
+唇
+唉
+唎
+唏
+唐
+唑
+唔
+唛
+唠
+唢
+唤
+唦
+唧
+唬
+售
+唯
+唰
+唱
+唳
+唵
+唷
+唻
+唼
+唾
+唿
+啁
+啃
+啄
+啅
+商
+啉
+啊
+啐
+啓
+啕
+啖
+啜
+啝
+啡
+啤
+啥
+啦
+啧
+啪
+啫
+啬
+啭
+啮
+啯
+啰
+啱
+啲
+啵
+啶
+啷
+啸
+啻
+啼
+啾
+喀
+喁
+喂
+喃
+善
+喆
+喇
+喈
+喉
+喊
+喋
+喎
+喏
+喑
+喔
+喘
+喙
+喜
+喝
+喟
+喦
+喧
+喰
+喱
+喳
+喵
+営
+喷
+喹
+喺
+喻
+喽
+喾
+嗄
+嗅
+嗉
+嗌
+嗍
+嗑
+嗒
+嗓
+嗔
+嗖
+嗜
+嗝
+嗞
+嗟
+嗡
+嗣
+嗤
+嗥
+嗦
+嗨
+嗪
+嗫
+嗬
+嗮
+嗯
+嗰
+嗲
+嗳
+嗵
+嗷
+嗽
+嗾
+嘀
+嘁
+嘅
+嘈
+嘉
+嘌
+嘎
+嘏
+嘘
+嘚
+嘛
+嘞
+嘟
+嘢
+嘣
+嘤
+嘦
+嘧
+嘬
+嘭
+嘱
+嘲
+嘴
+嘶
+嘹
+嘻
+嘿
+噁
+噉
+噌
+噎
+噐
+噔
+噗
+噘
+噙
+噜
+噢
+噤
+器
+噩
+噪
+噫
+噬
+噱
+噶
+噻
+噼
+嚅
+嚈
+嚎
+嚏
+嚐
+嚒
+嚓
+嚟
+嚣
+嚧
+嚩
+嚭
+嚯
+嚷
+嚼
+囊
+囍
+囔
+囖
+囗
+囘
+囚
+四
+囝
+回
+囟
+因
+囡
+团
+団
+囤
+囧
+囫
+囬
+园
+囯
+困
+囱
+囲
+図
+围
+囵
+囷
+囹
+固
+国
+图
+囿
+圃
+圄
+圆
+圈
+圉
+圊
+國
+圌
+圏
+圜
+圝
+圞
+土
+圣
+圧
+在
+圩
+圪
+圬
+圭
+圮
+圯
+地
+圳
+圹
+场
+圻
+圾
+址
+坂
+均
+坊
+坌
+坍
+坎
+坏
+坐
+坑
+块
+坚
+坛
+坜
+坝
+坞
+坟
+坠
+坡
+坤
+坦
+坨
+坩
+坪
+坫
+坬
+坭
+坯
+坳
+坷
+坻
+坼
+垂
+垃
+垄
+垅
+垆
+型
+垌
+垍
+垒
+垓
+垕
+垚
+垛
+垞
+垟
+垠
+垡
+垢
+垣
+垤
+垦
+垧
+垩
+垫
+垭
+垮
+垱
+垲
+垴
+垵
+垸
+埂
+埃
+埇
+埈
+埋
+埌
+城
+埏
+埒
+埔
+埕
+埗
+埘
+埙
+埚
+埜
+埝
+域
+埠
+埤
+埭
+埯
+埴
+埵
+埸
+培
+基
+埼
+埽
+堀
+堂
+堃
+堆
+堇
+堈
+堉
+堋
+堌
+堍
+堑
+堕
+堙
+堞
+堠
+堡
+堤
+堨
+堪
+堰
+堵
+堺
+堽
+塁
+塄
+塅
+塆
+塌
+塍
+塑
+塔
+塘
+塚
+塝
+塞
+塩
+填
+塬
+塭
+塱
+塽
+塾
+墀
+墁
+境
+墅
+墉
+墒
+墓
+墕
+増
+墘
+墙
+增
+墟
+墨
+墩
+墫
+壁
+壅
+壆
+壊
+壑
+壕
+壤
+士
+壬
+壮
+声
+壱
+売
+壳
+壶
+壸
+壹
+处
+备
+変
+复
+夏
+夑
+夔
+夕
+外
+夙
+多
+夜
+够
+夤
+夥
+大
+夨
+天
+太
+夫
+夬
+夭
+央
+夯
+失
+夲
+头
+夶
+夷
+夸
+夹
+夺
+夼
+奀
+奁
+奂
+奄
+奇
+奈
+奉
+奋
+奌
+奎
+奏
+契
+奔
+奕
+奖
+套
+奘
+奚
+奠
+奢
+奥
+奨
+奭
+女
+奴
+奶
+奸
+她
+好
+妁
+如
+妃
+妄
+妆
+妇
+妈
+妊
+妍
+妒
+妓
+妖
+妗
+妘
+妙
+妞
+妠
+妣
+妤
+妥
+妨
+妩
+妪
+妫
+妬
+妮
+妯
+妲
+妹
+妺
+妻
+妼
+妾
+姁
+姆
+姈
+姉
+姊
+始
+姐
+姑
+姒
+姓
+委
+姗
+姘
+姚
+姜
+姝
+姞
+姣
+姤
+姥
+姨
+姫
+姬
+姮
+姱
+姵
+姹
+姻
+姽
+姿
+威
+娃
+娄
+娅
+娆
+娇
+娈
+娉
+娌
+娑
+娓
+娘
+娜
+娟
+娠
+娡
+娣
+娥
+娩
+娭
+娱
+娲
+娴
+娶
+娼
+婀
+婆
+婉
+婊
+婕
+婚
+婠
+婢
+婧
+婪
+婬
+婳
+婴
+婵
+婶
+婷
+婺
+婻
+婼
+婿
+媄
+媒
+媗
+媚
+媛
+媜
+媞
+媪
+媲
+媳
+媵
+媸
+媺
+媾
+嫁
+嫂
+嫄
+嫉
+嫌
+嫒
+嫔
+嫖
+嫘
+嫚
+嫠
+嫡
+嫣
+嫦
+嫩
+嫪
+嫫
+嫰
+嫱
+嫲
+嫽
+嬅
+嬉
+嬖
+嬗
+嬛
+嬜
+嬢
+嬲
+嬴
+嬷
+嬾
+嬿
+孀
+子
+孑
+孒
+孓
+孔
+孕
+孖
+字
+存
+孙
+孚
+孛
+孜
+孝
+孟
+孢
+季
+孤
+孥
+学
+孩
+孪
+孬
+孰
+孱
+孳
+孵
+孺
+孽
+宀
+宁
+它
+宄
+宅
+宇
+守
+安
+宋
+完
+宍
+宏
+宓
+宕
+宗
+官
+宙
+定
+宛
+宜
+宝
+实
+実
+宠
+审
+客
+宣
+室
+宥
+宦
+宪
+宫
+宬
+宰
+害
+宴
+宵
+家
+宸
+容
+宽
+宾
+宿
+寀
+寂
+寃
+寄
+寅
+密
+寇
+富
+寐
+寒
+寓
+寔
+寘
+寛
+寝
+寞
+察
+寡
+寤
+寥
+寨
+寮
+寯
+寰
+寳
+寸
+对
+寺
+寻
+导
+対
+寿
+封
+専
+射
+尅
+将
+尉
+尊
+對
+小
+尐
+少
+尒
+尓
+尔
+尕
+尖
+尘
+尙
+尚
+尛
+尜
+尝
+尢
+尤
+尧
+尨
+尪
+尬
+就
+尴
+尸
+尹
+尺
+尻
+尼
+尽
+尾
+尿
+局
+屁
+层
+屃
+屄
+居
+屈
+屉
+届
+屋
+屌
+屍
+屎
+屏
+屐
+屑
+展
+屙
+属
+屠
+屡
+屣
+履
+屦
+屮
+屯
+山
+屹
+屺
+屾
+屿
+岀
+岁
+岂
+岈
+岌
+岐
+岑
+岔
+岕
+岖
+岗
+岘
+岙
+岚
+岛
+岜
+岞
+岢
+岣
+岩
+岫
+岬
+岭
+岱
+岳
+岵
+岷
+岸
+岺
+岽
+岿
+峁
+峄
+峇
+峋
+峒
+峕
+峙
+峠
+峡
+峣
+峤
+峥
+峦
+峨
+峩
+峪
+峭
+峯
+峰
+峻
+崀
+崁
+崂
+崃
+崄
+崆
+崇
+崎
+崐
+崑
+崔
+崖
+崚
+崛
+崞
+崟
+崤
+崦
+崧
+崩
+崭
+崮
+崴
+崽
+崾
+嵇
+嵊
+嵋
+嵌
+嵎
+嵖
+嵗
+嵘
+嵚
+嵛
+嵝
+嵩
+嵬
+嵯
+嵴
+嶂
+嶋
+嶙
+嶝
+嶲
+嶷
+巂
+巅
+巇
+巉
+巍
+巎
+巘
+巜
+川
+州
+巡
+巢
+巣
+工
+左
+巧
+巨
+巩
+巫
+差
+巯
+己
+已
+巳
+巴
+巷
+巻
+巽
+巾
+巿
+币
+市
+布
+帅
+帆
+师
+希
+帏
+帐
+帑
+帔
+帕
+帖
+帘
+帙
+帚
+帛
+帜
+帝
+带
+帧
+席
+帮
+帯
+帰
+帷
+常
+帻
+帼
+帽
+幂
+幄
+幅
+幌
+幔
+幕
+幛
+幞
+幡
+幢
+干
+平
+年
+幵
+并
+幷
+幸
+幺
+幻
+幼
+幽
+广
+庀
+庁
+広
+庄
+庆
+庇
+床
+庋
+序
+庐
+庑
+库
+应
+底
+庖
+店
+庙
+庚
+府
+庞
+废
+庠
+庤
+庥
+度
+座
+庭
+庵
+庶
+康
+庸
+庹
+庾
+廆
+廉
+廊
+廋
+廌
+廑
+廒
+廓
+廕
+廖
+廙
+廛
+廞
+廨
+廪
+廯
+延
+廷
+建
+廻
+廼
+廾
+廿
+开
+弁
+异
+弃
+弄
+弇
+弈
+弊
+弋
+式
+弐
+弑
+弓
+引
+弗
+弘
+弛
+弟
+张
+弢
+弥
+弦
+弧
+弩
+弭
+弯
+弱
+弹
+强
+弼
+弾
+彀
+归
+当
+录
+彖
+彗
+彘
+彝
+彟
+彡
+形
+彤
+彦
+彧
+彩
+彪
+彬
+彭
+彰
+影
+彳
+彵
+彷
+役
+彻
+彼
+往
+征
+徂
+径
+待
+徇
+很
+徉
+徊
+律
+徐
+徒
+従
+徕
+得
+徘
+徙
+徜
+御
+徧
+徨
+循
+徭
+微
+徳
+徴
+徵
+德
+徼
+徽
+心
+忄
+必
+忆
+忉
+忌
+忍
+忏
+忐
+忑
+忒
+忖
+志
+忘
+忙
+応
+忝
+忞
+忠
+忡
+忤
+忧
+忪
+快
+忭
+忱
+念
+忸
+忻
+忽
+忾
+忿
+怀
+态
+怂
+怃
+怄
+怅
+怆
+怍
+怎
+怏
+怒
+怔
+怕
+怖
+怙
+怛
+怜
+思
+怠
+怡
+急
+怦
+性
+怨
+怩
+怪
+怫
+怯
+怱
+怳
+怵
+怹
+总
+怼
+怿
+恁
+恂
+恃
+恋
+恍
+恏
+恐
+恒
+恕
+恙
+恚
+恠
+恢
+恣
+恤
+恨
+恩
+恪
+恫
+恬
+恭
+息
+恰
+恳
+恵
+恶
+恸
+恹
+恺
+恻
+恼
+恽
+恿
+悃
+悄
+悉
+悌
+悍
+悒
+悔
+悖
+悚
+悛
+悝
+悟
+悠
+患
+悦
+您
+悩
+悪
+悫
+悬
+悭
+悯
+悰
+悱
+悲
+悳
+悴
+悸
+悻
+悼
+情
+惆
+惇
+惊
+惋
+惑
+惔
+惕
+惘
+惚
+惛
+惜
+惝
+惟
+惠
+惢
+惣
+惦
+惧
+惨
+惩
+惪
+惫
+惬
+惭
+惮
+惯
+惰
+想
+惴
+惶
+惹
+惺
+愀
+愁
+愆
+愈
+愉
+愍
+愎
+意
+愔
+愕
+愚
+感
+愠
+愣
+愤
+愦
+愧
+愫
+愬
+愰
+愽
+愿
+慆
+慈
+慊
+慌
+慎
+慑
+慕
+慜
+慝
+慢
+慥
+慧
+慨
+慰
+慵
+慷
+慾
+憋
+憍
+憎
+憔
+憙
+憧
+憨
+憩
+憬
+憷
+憺
+憾
+懂
+懈
+懊
+懋
+懐
+懑
+懒
+懔
+懦
+懮
+懵
+懽
+懿
+戆
+戈
+戊
+戋
+戌
+戍
+戎
+戏
+成
+我
+戒
+戓
+戕
+或
+戗
+战
+戚
+戛
+戟
+戡
+戢
+戥
+戦
+截
+戬
+戮
+戯
+戳
+戴
+户
+戻
+戽
+戾
+房
+所
+扁
+扃
+扆
+扇
+扈
+扉
+手
+扌
+才
+扎
+扑
+扒
+打
+扔
+托
+扛
+扞
+扣
+扥
+扦
+执
+扩
+扪
+扫
+扬
+扭
+扮
+扯
+扰
+扳
+扶
+批
+扼
+扽
+找
+承
+技
+抃
+抄
+抉
+把
+抑
+抒
+抓
+抔
+投
+抖
+抗
+折
+抚
+抛
+抜
+抟
+抠
+抡
+抢
+护
+报
+抧
+抨
+披
+抬
+抱
+抳
+抵
+抹
+抺
+抻
+押
+抽
+抿
+拂
+拄
+担
+拆
+拇
+拈
+拉
+拊
+拌
+拍
+拎
+拏
+拐
+拒
+拓
+拔
+拖
+拗
+拘
+拙
+招
+拜
+拟
+拢
+拣
+拥
+拦
+拧
+拨
+择
+括
+拭
+拮
+拯
+拱
+拳
+拴
+拶
+拷
+拼
+拽
+拾
+拿
+挀
+持
+挂
+指
+挈
+按
+挎
+挑
+挒
+挖
+挚
+挛
+挝
+挞
+挟
+挠
+挡
+挢
+挣
+挤
+挥
+挨
+挪
+挫
+振
+挲
+挹
+挺
+挻
+挼
+挽
+捂
+捃
+捅
+捆
+捉
+捋
+捌
+捍
+捎
+捏
+捐
+捕
+捜
+捞
+损
+捡
+换
+捣
+捧
+捩
+捭
+据
+捯
+捱
+捶
+捷
+捺
+捻
+捽
+掀
+掂
+掇
+授
+掉
+掊
+掌
+掎
+掏
+掐
+排
+掖
+掘
+掞
+掠
+探
+掣
+掤
+接
+控
+推
+掩
+措
+掬
+掭
+掮
+掰
+掲
+掳
+掴
+掷
+掸
+掺
+掼
+掾
+揄
+揆
+揉
+揍
+描
+提
+插
+揖
+揠
+握
+揣
+揩
+揪
+揭
+揲
+援
+揵
+揶
+揸
+揺
+揽
+揾
+揿
+搀
+搁
+搂
+搅
+搋
+搏
+搐
+搓
+搔
+搜
+搞
+搠
+搡
+搢
+搥
+搦
+搧
+搨
+搪
+搬
+搭
+搴
+搵
+携
+搽
+搿
+摁
+摄
+摅
+摆
+摇
+摈
+摊
+摒
+摔
+摘
+摛
+摞
+摧
+摩
+摭
+摸
+摹
+摺
+摽
+撂
+撃
+撄
+撅
+撇
+撑
+撒
+撕
+撘
+撙
+撝
+撞
+撤
+撩
+撬
+播
+撮
+撰
+撵
+撷
+撸
+撺
+撼
+擀
+擂
+擅
+操
+擎
+擒
+擗
+擘
+擞
+擢
+擤
+擦
+擫
+擿
+攀
+攒
+攘
+攞
+攥
+攫
+支
+攲
+攴
+攵
+收
+攸
+改
+攻
+攽
+放
+政
+故
+效
+敉
+敌
+敎
+敏
+救
+敔
+敕
+敖
+教
+敚
+敛
+敝
+敞
+敢
+散
+敦
+敫
+敬
+数
+敲
+整
+敷
+敻
+文
+斉
+斋
+斌
+斎
+斐
+斑
+斓
+斗
+料
+斛
+斜
+斝
+斟
+斡
+斤
+斥
+斧
+斩
+斫
+断
+斯
+新
+斲
+斶
+方
+於
+施
+旁
+旃
+旄
+旅
+旆
+旋
+旌
+旎
+族
+旒
+旖
+旗
+旛
+无
+既
+旣
+日
+旦
+旧
+旨
+早
+旬
+旭
+旮
+旯
+旰
+旱
+旳
+旴
+时
+旷
+旸
+旺
+旻
+旼
+昀
+昂
+昃
+昆
+昇
+昉
+昊
+昌
+明
+昏
+易
+昔
+昕
+昙
+昚
+昝
+昞
+星
+映
+春
+昧
+昨
+昪
+昫
+昭
+是
+昰
+昱
+昳
+昴
+昵
+昶
+昺
+昼
+显
+晁
+時
+晃
+晄
+晋
+晌
+晏
+晒
+晓
+晔
+晕
+晖
+晗
+晙
+晚
+晞
+晟
+晡
+晢
+晤
+晦
+晧
+晨
+晩
+晬
+普
+景
+晰
+晳
+晴
+晶
+晷
+晸
+智
+晻
+晾
+暁
+暂
+暄
+暇
+暌
+暍
+暎
+暐
+暑
+暕
+暖
+暗
+暝
+暠
+暧
+暨
+暮
+暴
+暸
+暹
+暻
+暾
+曈
+曌
+曕
+曙
+曛
+曜
+曝
+曦
+曩
+曰
+曱
+曲
+曳
+更
+曷
+曹
+曺
+曼
+曽
+曾
+替
+最
+會
+朅
+月
+有
+朊
+朋
+服
+朏
+朐
+朓
+朔
+朕
+朗
+望
+朝
+期
+朥
+朦
+木
+未
+末
+本
+札
+术
+朱
+朲
+朴
+朵
+机
+朽
+朿
+杀
+杂
+权
+杆
+杈
+杉
+杌
+李
+杏
+材
+村
+杓
+杖
+杜
+杞
+束
+杠
+条
+来
+杧
+杨
+杪
+杭
+杮
+杯
+杰
+杲
+杳
+杵
+杷
+杼
+松
+板
+极
+构
+枇
+枉
+枋
+枏
+析
+枓
+枕
+林
+枘
+枚
+果
+枝
+枞
+枟
+枠
+枢
+枣
+枥
+枧
+枨
+枪
+枫
+枭
+枯
+枰
+枱
+枳
+枵
+架
+枷
+枸
+枹
+柁
+柃
+柄
+柊
+柏
+某
+柑
+柒
+染
+柔
+柘
+柙
+柚
+柜
+柝
+柞
+柟
+柠
+柢
+查
+柩
+柬
+柯
+柰
+柱
+柳
+柴
+柷
+査
+柽
+柾
+柿
+栀
+栃
+栄
+栅
+标
+栈
+栉
+栊
+栋
+栌
+栎
+栏
+树
+栒
+栓
+栖
+栗
+栝
+栞
+栟
+校
+栢
+栩
+株
+栱
+栲
+栳
+栴
+样
+核
+根
+栻
+格
+栽
+栾
+栿
+桀
+桁
+桂
+桃
+桄
+桅
+框
+案
+桉
+桌
+桎
+桐
+桑
+桓
+桔
+桕
+桖
+桜
+桠
+桡
+桢
+档
+桤
+桥
+桦
+桧
+桨
+桩
+桫
+桴
+桶
+桷
+梁
+梃
+梅
+梆
+梏
+梓
+梗
+梢
+梣
+梦
+梧
+梨
+梭
+梯
+械
+梳
+梵
+梶
+梼
+梿
+检
+棂
+棉
+棋
+棍
+棐
+棒
+棕
+棘
+棚
+棠
+棣
+棨
+棪
+棫
+森
+棰
+棱
+棵
+棹
+棺
+棻
+棼
+椀
+椁
+椅
+椇
+椋
+植
+椎
+椐
+椒
+椛
+検
+椟
+椠
+椤
+椪
+椭
+椰
+椴
+椹
+椽
+椿
+楂
+楔
+楗
+楙
+楚
+楛
+楝
+楞
+楠
+楢
+楣
+楤
+楦
+楩
+楪
+楫
+業
+楮
+楯
+楶
+楷
+楸
+楹
+楼
+楽
+榀
+概
+榃
+榄
+榆
+榇
+榈
+榉
+榊
+榎
+榔
+榕
+榖
+榘
+榛
+榜
+榧
+榨
+榫
+榭
+榱
+榴
+榷
+榻
+榼
+槁
+槃
+槅
+槊
+槌
+槎
+槐
+槑
+槔
+様
+槙
+槚
+槛
+槜
+槟
+槠
+槭
+槱
+槲
+槵
+槻
+槽
+槿
+樊
+樋
+樗
+樘
+樛
+樟
+模
+樨
+権
+横
+樫
+樯
+樱
+樵
+樽
+樾
+橄
+橇
+橐
+橘
+橙
+橚
+橛
+橡
+橥
+橦
+橱
+橹
+橼
+檀
+檄
+檇
+檎
+檐
+檗
+檞
+檠
+檩
+檫
+檬
+檵
+櫂
+櫆
+欠
+次
+欢
+欣
+欤
+欧
+欲
+欷
+欸
+欹
+欺
+欻
+款
+歃
+歆
+歇
+歉
+歌
+歔
+歘
+歙
+止
+正
+此
+步
+武
+歧
+歩
+歪
+歳
+歴
+歹
+歺
+死
+歼
+殁
+殂
+殃
+殄
+殆
+殇
+殉
+殊
+残
+殍
+殑
+殒
+殓
+殖
+殚
+殛
+殡
+殢
+殪
+殳
+殴
+段
+殷
+殿
+毁
+毂
+毅
+毋
+毌
+母
+毎
+每
+毐
+毑
+毒
+毓
+比
+毕
+毖
+毗
+毘
+毙
+毛
+毡
+毫
+毯
+毳
+毵
+毽
+氀
+氂
+氅
+氆
+氇
+氍
+氏
+氐
+民
+氓
+气
+氖
+気
+氘
+氙
+氚
+氛
+氟
+氡
+氢
+氤
+氦
+氧
+氨
+氩
+氪
+氮
+氯
+氰
+氲
+水
+氵
+氷
+永
+氹
+氺
+氽
+氿
+汀
+汁
+求
+汆
+汇
+汉
+汊
+汐
+汕
+汖
+汗
+汛
+汜
+汝
+汞
+江
+池
+污
+汤
+汧
+汨
+汩
+汪
+汭
+汰
+汲
+汴
+汶
+汸
+汹
+汽
+汾
+沁
+沂
+沃
+沄
+沅
+沆
+沇
+沈
+沉
+沌
+沏
+沐
+沒
+沓
+沔
+沕
+沙
+沚
+沛
+沟
+没
+沢
+沣
+沤
+沥
+沦
+沧
+沨
+沩
+沪
+沫
+沬
+沭
+沮
+沱
+河
+沴
+沵
+沸
+油
+治
+沼
+沽
+沾
+沿
+泃
+泄
+泅
+泇
+泉
+泊
+泌
+泐
+泓
+泔
+法
+泖
+泗
+泚
+泛
+泞
+泠
+泡
+波
+泣
+泥
+注
+泩
+泪
+泫
+泮
+泯
+泰
+泱
+泳
+泵
+泷
+泸
+泺
+泻
+泼
+泽
+泾
+洁
+洄
+洇
+洈
+洊
+洋
+洌
+洎
+洑
+洒
+洗
+洙
+洛
+洞
+洢
+洣
+津
+洧
+洨
+洪
+洫
+洮
+洱
+洲
+洳
+洵
+洸
+洹
+洺
+活
+洼
+洽
+派
+流
+浃
+浄
+浅
+浆
+浇
+浈
+浉
+浊
+测
+浍
+济
+浏
+浐
+浑
+浒
+浓
+浔
+浙
+浚
+浛
+浜
+浞
+浠
+浡
+浣
+浥
+浦
+浩
+浪
+浬
+浮
+浯
+浴
+海
+浸
+浼
+涂
+涅
+消
+涉
+涌
+涎
+涐
+涑
+涓
+涔
+涕
+涘
+涙
+涛
+涝
+涞
+涟
+涠
+涡
+涢
+涣
+涤
+润
+涧
+涨
+涩
+涪
+涫
+涮
+涯
+液
+涴
+涵
+涸
+涿
+淀
+淄
+淅
+淆
+淇
+淋
+淌
+淏
+淑
+淖
+淘
+淙
+淛
+淝
+淞
+淠
+淡
+淤
+淦
+淩
+淫
+淬
+淮
+淯
+深
+淳
+混
+淸
+淹
+添
+淼
+渀
+渃
+清
+済
+渉
+渊
+渋
+渌
+渍
+渎
+渐
+渑
+渔
+渕
+渖
+渗
+渚
+渝
+渟
+渠
+渡
+渣
+渤
+渥
+温
+渫
+渭
+港
+渲
+渴
+游
+渺
+渼
+湃
+湄
+湉
+湋
+湍
+湎
+湑
+湓
+湔
+湖
+湘
+湛
+湜
+湟
+湣
+湫
+湮
+湲
+湳
+湴
+湾
+湿
+満
+溁
+溃
+溅
+溆
+溇
+溉
+溍
+溏
+源
+溘
+溜
+溞
+溟
+溢
+溥
+溦
+溧
+溪
+溯
+溱
+溲
+溴
+溶
+溷
+溺
+溽
+滁
+滂
+滃
+滆
+滇
+滈
+滉
+滋
+滍
+滏
+滑
+滓
+滔
+滕
+滗
+滘
+滙
+滚
+滝
+滞
+滟
+滠
+满
+滢
+滤
+滥
+滦
+滨
+滩
+滴
+滹
+漀
+漂
+漆
+漈
+漉
+漏
+漓
+演
+漕
+漠
+漩
+漪
+漫
+漭
+漯
+漱
+漳
+漶
+漷
+漾
+潆
+潇
+潋
+潍
+潏
+潘
+潜
+潞
+潟
+潢
+潦
+潭
+潮
+潲
+潴
+潸
+潺
+潼
+潽
+潾
+澂
+澄
+澈
+澉
+澌
+澍
+澎
+澐
+澔
+澜
+澡
+澥
+澧
+澪
+澳
+澶
+澹
+激
+濂
+濆
+濉
+濊
+濑
+濒
+濙
+濛
+濞
+濠
+濡
+濩
+濬
+濮
+濯
+瀀
+瀍
+瀑
+瀚
+瀛
+瀞
+瀣
+瀬
+瀹
+瀼
+灌
+灏
+灞
+火
+灬
+灭
+灯
+灰
+灵
+灶
+灸
+灼
+灾
+灿
+炀
+炁
+炅
+炆
+炉
+炊
+炎
+炒
+炔
+炕
+炖
+炘
+炙
+炜
+炝
+炟
+炤
+炩
+炫
+炬
+炭
+炮
+炯
+炱
+炳
+炷
+炸
+点
+為
+炻
+炼
+炽
+烀
+烁
+烂
+烃
+烈
+烊
+烎
+烔
+烘
+烙
+烛
+烜
+烝
+烟
+烤
+烦
+烧
+烨
+烩
+烫
+烬
+热
+烯
+烷
+烹
+烺
+烽
+焉
+焊
+焌
+焐
+焓
+焕
+焖
+焗
+焘
+焙
+焚
+焜
+焞
+焦
+焮
+焯
+焰
+焱
+然
+焼
+煅
+煇
+煊
+煌
+煎
+煐
+煕
+煖
+煚
+煜
+煞
+煤
+煦
+照
+煨
+煮
+煲
+煳
+煴
+煸
+煺
+煽
+熄
+熇
+熊
+熏
+熔
+熘
+熙
+熜
+熟
+熠
+熥
+熨
+熬
+熳
+熵
+熹
+熺
+燃
+燊
+燋
+燎
+燏
+燔
+燕
+燚
+燠
+燥
+燧
+燮
+燹
+燻
+燿
+爀
+爆
+爇
+爨
+爪
+爬
+爰
+爱
+爲
+爵
+父
+爷
+爸
+爹
+爻
+爽
+爿
+牀
+牁
+牂
+片
+版
+牋
+牌
+牍
+牐
+牒
+牖
+牙
+牛
+牝
+牟
+牠
+牡
+牢
+牤
+牦
+牧
+物
+牯
+牲
+牵
+特
+牺
+牻
+牾
+犀
+犁
+犄
+犇
+犊
+犍
+犏
+犒
+犟
+犨
+犬
+犭
+犯
+犰
+犴
+状
+犷
+犸
+犹
+犼
+犽
+狁
+狂
+狃
+狄
+狈
+狌
+狍
+狎
+狐
+狒
+狗
+狙
+狛
+狝
+狞
+狠
+狡
+狨
+狩
+独
+狭
+狮
+狯
+狰
+狱
+狲
+狳
+狴
+狷
+狸
+狻
+狼
+猀
+猁
+猃
+猄
+猇
+猊
+猋
+猎
+猓
+猕
+猖
+猗
+猛
+猜
+猝
+猞
+猟
+猡
+猢
+猥
+猩
+猪
+猫
+猬
+献
+猰
+猱
+猴
+猷
+猹
+猾
+猿
+獍
+獐
+獒
+獗
+獠
+獣
+獬
+獭
+獴
+獾
+玁
+玄
+率
+玉
+王
+玎
+玏
+玑
+玕
+玖
+玗
+玘
+玙
+玚
+玛
+玟
+玠
+玡
+玢
+玥
+玦
+玧
+玩
+玫
+玭
+玮
+环
+现
+玲
+玳
+玷
+玹
+玺
+玻
+珀
+珂
+珅
+珈
+珉
+珊
+珍
+珏
+珐
+珑
+珖
+珙
+珝
+珞
+珠
+珣
+珥
+珦
+珧
+珩
+珪
+班
+珰
+珲
+珵
+珹
+珺
+珽
+琀
+球
+琅
+理
+琇
+琉
+琊
+琋
+琍
+琎
+琏
+琐
+琚
+琛
+琢
+琤
+琥
+琦
+琨
+琪
+琬
+琮
+琯
+琰
+琲
+琳
+琴
+琵
+琶
+琹
+琼
+瑀
+瑁
+瑄
+瑆
+瑊
+瑒
+瑕
+瑗
+瑙
+瑚
+瑛
+瑜
+瑞
+瑟
+瑠
+瑢
+瑧
+瑨
+瑭
+瑰
+瑱
+瑶
+瑷
+瑸
+瑺
+瑾
+璀
+璁
+璂
+璃
+璆
+璇
+璈
+璋
+璎
+璐
+璘
+璜
+璞
+璟
+璠
+璧
+璨
+璩
+璪
+璮
+璲
+璺
+璿
+瓌
+瓒
+瓘
+瓛
+瓜
+瓞
+瓟
+瓠
+瓢
+瓣
+瓤
+瓦
+瓮
+瓯
+瓴
+瓶
+瓷
+瓿
+甃
+甄
+甍
+甏
+甑
+甓
+甗
+甘
+甙
+甚
+甜
+生
+甡
+產
+甥
+用
+甩
+甪
+甫
+甬
+甭
+甯
+田
+由
+甲
+申
+电
+男
+甸
+町
+画
+甾
+畀
+畅
+畈
+畊
+畋
+界
+畎
+畏
+畑
+畔
+留
+畚
+畛
+畜
+畠
+畤
+略
+畦
+番
+畯
+畲
+畴
+畸
+畹
+畿
+疃
+疆
+疋
+疍
+疎
+疏
+疑
+疒
+疔
+疖
+疗
+疙
+疚
+疝
+疟
+疠
+疡
+疣
+疤
+疥
+疫
+疬
+疮
+疯
+疰
+疱
+疲
+疳
+疴
+疵
+疸
+疹
+疼
+疽
+疾
+痂
+痄
+病
+症
+痈
+痉
+痊
+痍
+痒
+痔
+痕
+痖
+痘
+痛
+痞
+痢
+痣
+痤
+痦
+痧
+痨
+痩
+痪
+痫
+痰
+痱
+痴
+痹
+痼
+痿
+瘀
+瘁
+瘅
+瘆
+瘊
+瘌
+瘐
+瘕
+瘖
+瘗
+瘘
+瘙
+瘛
+瘟
+瘠
+瘢
+瘤
+瘥
+瘦
+瘩
+瘪
+瘫
+瘰
+瘳
+瘴
+瘵
+瘸
+瘼
+瘾
+瘿
+癀
+癃
+癌
+癍
+癎
+癒
+癔
+癖
+癜
+癞
+癣
+癫
+癯
+癸
+発
+登
+發
+白
+百
+癿
+皂
+的
+皆
+皇
+皈
+皋
+皎
+皐
+皑
+皒
+皓
+皕
+皖
+皙
+皛
+皝
+皞
+皤
+皦
+皮
+皱
+皲
+皴
+皿
+盂
+盃
+盅
+盆
+盈
+盉
+益
+盌
+盍
+盎
+盏
+盐
+监
+盒
+盔
+盖
+盗
+盘
+盛
+盝
+盟
+盥
+盦
+盨
+盩
+目
+盯
+盱
+盲
+直
+相
+盹
+盼
+盾
+眀
+省
+眄
+眇
+眈
+眉
+看
+県
+眙
+眚
+眛
+眞
+真
+眠
+眦
+眨
+眩
+眬
+眭
+眯
+眵
+眶
+眷
+眸
+眺
+眼
+着
+睁
+睇
+睐
+睑
+睒
+睚
+睛
+睡
+睢
+督
+睥
+睦
+睨
+睪
+睫
+睬
+睱
+睹
+睺
+睽
+睾
+睿
+瞀
+瞄
+瞅
+瞋
+瞌
+瞎
+瞑
+瞒
+瞟
+瞠
+瞢
+瞥
+瞧
+瞩
+瞪
+瞬
+瞭
+瞰
+瞳
+瞻
+瞽
+瞿
+矍
+矗
+矛
+矜
+矞
+矢
+矣
+知
+矧
+矩
+矫
+矬
+短
+矮
+石
+矶
+矸
+矽
+矾
+矿
+砀
+码
+砂
+砉
+砌
+砍
+砑
+砒
+研
+砕
+砖
+砗
+砚
+砜
+砝
+砟
+砢
+砣
+砥
+砦
+砧
+砩
+砬
+砭
+砰
+砳
+破
+砵
+砷
+砸
+砹
+砺
+砻
+砼
+砾
+础
+硅
+硇
+硌
+硎
+硏
+硐
+硒
+硔
+硕
+硖
+硗
+硙
+硚
+硝
+硪
+硫
+硬
+确
+硷
+硼
+碁
+碇
+碉
+碌
+碍
+碎
+碏
+碑
+碓
+碗
+碘
+碚
+碛
+碜
+碟
+碡
+碣
+碥
+碧
+碰
+碱
+碲
+碳
+碴
+碶
+碹
+碾
+磁
+磅
+磉
+磊
+磋
+磐
+磔
+磕
+磙
+磜
+磡
+磦
+磨
+磬
+磲
+磴
+磷
+磺
+磻
+磾
+礁
+礅
+礐
+礓
+礞
+礤
+礴
+示
+礻
+礼
+礽
+社
+祀
+祁
+祂
+祆
+祇
+祈
+祉
+祊
+祎
+祏
+祐
+祓
+祔
+祖
+祗
+祘
+祚
+祛
+祜
+祝
+神
+祟
+祠
+祢
+祥
+祧
+票
+祭
+祯
+祲
+祷
+祸
+祹
+祺
+祼
+祾
+禀
+禁
+禄
+禅
+禇
+禊
+禋
+福
+禑
+禔
+禖
+禘
+禚
+禛
+禟
+禤
+禧
+禩
+禳
+禵
+禹
+禺
+离
+禽
+禾
+秀
+私
+秂
+秃
+秆
+秉
+秋
+种
+科
+秒
+秕
+秘
+租
+秣
+秤
+秦
+秧
+秩
+秫
+秬
+秭
+积
+称
+秸
+移
+秽
+秾
+稀
+稃
+程
+稍
+税
+稔
+稗
+稙
+稚
+稞
+稠
+稣
+稲
+稳
+稷
+稹
+稻
+稼
+稽
+稿
+穂
+穆
+穉
+穏
+穑
+穗
+穣
+穰
+穴
+究
+穷
+穹
+空
+穿
+突
+窃
+窄
+窅
+窆
+窈
+窊
+窋
+窍
+窑
+窒
+窓
+窕
+窖
+窗
+窘
+窜
+窝
+窟
+窠
+窣
+窥
+窦
+窨
+窭
+窰
+窳
+窸
+窿
+立
+竑
+竖
+站
+竜
+竝
+竞
+竟
+章
+竣
+童
+竦
+竭
+端
+竹
+竺
+竽
+竿
+笃
+笄
+笆
+笈
+笊
+笋
+笏
+笑
+笔
+笕
+笙
+笛
+笞
+笠
+笤
+笥
+符
+笨
+笪
+笫
+第
+笮
+笱
+笳
+笸
+笹
+笺
+笼
+笾
+筅
+筇
+等
+筊
+筋
+筌
+筏
+筐
+筑
+筒
+答
+策
+筘
+筚
+筛
+筜
+筝
+筠
+筭
+筮
+筯
+筱
+筲
+筴
+筵
+筷
+筹
+筼
+签
+简
+箅
+箌
+箍
+箐
+箓
+箔
+箕
+算
+箜
+箝
+管
+箦
+箧
+箨
+箩
+箪
+箫
+箬
+箭
+箱
+箴
+箸
+篁
+篆
+篇
+篌
+篑
+篓
+篙
+篚
+篝
+篡
+篥
+篦
+篪
+篮
+篯
+篱
+篷
+篼
+篾
+簃
+簇
+簋
+簌
+簏
+簕
+簖
+簟
+簠
+簦
+簧
+簪
+簰
+簸
+簿
+籀
+籁
+籍
+籓
+籙
+米
+籴
+籺
+类
+籼
+籽
+粄
+粉
+粑
+粒
+粕
+粗
+粘
+粜
+粝
+粞
+粟
+粢
+粤
+粥
+粦
+粧
+粪
+粬
+粮
+粱
+粲
+粳
+粹
+粼
+粽
+精
+粿
+糁
+糅
+糊
+糌
+糍
+糕
+糖
+糗
+糙
+糜
+糟
+糠
+糨
+糬
+糯
+糸
+系
+紊
+紘
+素
+索
+紧
+紫
+紬
+紮
+累
+経
+絜
+絪
+絮
+絵
+絶
+絷
+絺
+綎
+綖
+継
+続
+綝
+綦
+綫
+綮
+総
+緑
+緾
+縁
+縂
+縠
+縢
+縻
+繁
+繇
+繋
+繸
+繻
+纁
+纂
+纚
+纛
+纟
+纠
+纡
+红
+纣
+纤
+纥
+约
+级
+纨
+纩
+纪
+纫
+纬
+纭
+纮
+纯
+纰
+纱
+纲
+纳
+纵
+纶
+纷
+纸
+纹
+纺
+纻
+纽
+纾
+线
+绀
+绁
+绂
+练
+组
+绅
+细
+织
+终
+绉
+绊
+绌
+绍
+绎
+经
+绐
+绑
+绒
+结
+绔
+绕
+绗
+绘
+给
+绚
+绛
+络
+绝
+绞
+统
+绠
+绡
+绢
+绣
+绥
+绦
+继
+绨
+绩
+绪
+绫
+续
+绮
+绯
+绰
+绱
+绲
+绳
+维
+绵
+绶
+绷
+绸
+绹
+绺
+绻
+综
+绽
+绾
+绿
+缀
+缁
+缂
+缃
+缄
+缅
+缆
+缇
+缈
+缉
+缊
+缋
+缌
+缍
+缎
+缐
+缑
+缒
+缓
+缔
+缕
+编
+缗
+缘
+缙
+缚
+缛
+缜
+缝
+缟
+缠
+缡
+缢
+缣
+缤
+缥
+缦
+缧
+缨
+缩
+缪
+缫
+缬
+缭
+缮
+缯
+缰
+缱
+缲
+缳
+缴
+缵
+缶
+缷
+缸
+缺
+罂
+罃
+罄
+罅
+罍
+罐
+网
+罔
+罕
+罗
+罘
+罚
+罝
+罟
+罠
+罡
+罢
+罣
+罥
+罨
+罩
+罪
+置
+罱
+署
+罴
+罹
+罽
+罾
+羁
+羊
+羌
+美
+羑
+羔
+羕
+羚
+羝
+羞
+羟
+羡
+羣
+群
+羧
+羮
+羯
+羰
+羲
+羸
+羹
+羼
+羽
+羿
+翀
+翁
+翃
+翅
+翈
+翊
+翌
+翎
+翔
+翕
+翘
+翙
+翚
+翛
+翟
+翠
+翡
+翥
+翦
+翩
+翫
+翮
+翰
+翱
+翳
+翻
+翼
+翾
+耀
+老
+考
+耄
+者
+耆
+耋
+而
+耍
+耐
+耒
+耔
+耕
+耗
+耘
+耙
+耜
+耦
+耧
+耨
+耩
+耪
+耳
+耵
+耶
+耷
+耸
+耻
+耽
+耿
+聂
+聃
+聆
+聊
+聋
+职
+聍
+聒
+联
+聕
+聘
+聚
+聡
+聩
+聪
+聴
+聼
+聿
+肃
+肄
+肆
+肇
+肉
+肋
+肌
+肓
+肖
+肘
+肚
+肛
+肜
+肝
+肟
+肠
+股
+肢
+肤
+肥
+肩
+肪
+肫
+肮
+肯
+肱
+育
+肴
+肸
+肺
+肼
+肽
+肾
+肿
+胀
+胁
+胂
+胃
+胄
+胆
+背
+胍
+胎
+胖
+胗
+胙
+胚
+胛
+胜
+胝
+胞
+胠
+胡
+胤
+胥
+胧
+胨
+胪
+胫
+胬
+胭
+胯
+胰
+胱
+胳
+胴
+胶
+胸
+胺
+胼
+能
+脁
+脂
+脆
+脇
+脉
+脊
+脍
+脏
+脐
+脑
+脒
+脓
+脔
+脖
+脘
+脚
+脞
+脢
+脩
+脬
+脯
+脱
+脲
+脳
+脷
+脸
+脾
+脿
+腆
+腈
+腊
+腋
+腌
+腐
+腑
+腓
+腔
+腕
+腘
+腙
+腚
+腠
+腥
+腧
+腩
+腭
+腮
+腰
+腱
+腴
+腹
+腺
+腻
+腼
+腾
+腿
+膀
+膂
+膈
+膊
+膏
+膑
+膘
+膛
+膜
+膝
+膦
+膨
+膳
+膺
+膻
+臀
+臁
+臂
+臃
+臆
+臊
+臌
+臑
+臓
+臜
+臞
+臣
+臧
+自
+臬
+臭
+至
+致
+臵
+臻
+臼
+臾
+舀
+舁
+舂
+舄
+舅
+舆
+舌
+舍
+舎
+舐
+舒
+舔
+舖
+舘
+舛
+舜
+舞
+舟
+舡
+舢
+舣
+舨
+航
+舫
+般
+舯
+舰
+舱
+舲
+舳
+舴
+舵
+舶
+舷
+舸
+船
+舺
+舻
+舾
+艄
+艇
+艉
+艋
+艏
+艘
+艟
+艨
+艮
+良
+艰
+色
+艳
+艹
+艺
+艽
+艾
+艿
+节
+芃
+芄
+芈
+芊
+芋
+芍
+芎
+芏
+芑
+芒
+芗
+芘
+芙
+芜
+芝
+芟
+芡
+芣
+芤
+芥
+芦
+芨
+芩
+芪
+芫
+芬
+芭
+芮
+芯
+芰
+花
+芳
+芴
+芶
+芷
+芸
+芹
+芽
+芾
+苁
+苄
+苇
+苈
+苊
+苋
+苌
+苍
+苎
+苏
+苑
+苒
+苓
+苔
+苕
+苗
+苘
+苛
+苜
+苞
+苟
+苡
+苢
+苣
+苤
+若
+苦
+苫
+苭
+苯
+英
+苴
+苷
+苹
+苺
+苻
+苼
+苾
+茀
+茁
+茂
+范
+茄
+茅
+茆
+茇
+茈
+茉
+茌
+茎
+茏
+茑
+茔
+茕
+茗
+茚
+茛
+茜
+茝
+茧
+茨
+茫
+茬
+茭
+茯
+茱
+茳
+茴
+茵
+茶
+茸
+茹
+茺
+茼
+荀
+荃
+荄
+荅
+荆
+荇
+荈
+草
+荏
+荐
+荑
+荒
+荔
+荖
+荘
+荚
+荛
+荜
+荞
+荟
+荠
+荡
+荣
+荤
+荥
+荦
+荧
+荨
+荩
+荪
+荫
+荬
+荭
+荮
+药
+荳
+荷
+荸
+荻
+荼
+荽
+莃
+莅
+莆
+莉
+莎
+莒
+莓
+莘
+莙
+莛
+莜
+莞
+莠
+莨
+莩
+莪
+莫
+莱
+莲
+莳
+莴
+莶
+获
+莸
+莹
+莺
+莼
+莽
+菀
+菁
+菂
+菅
+菇
+菈
+菉
+菊
+菌
+菏
+菑
+菓
+菔
+菖
+菘
+菜
+菝
+菟
+菠
+菡
+菩
+菪
+菫
+菰
+菱
+菲
+菸
+菹
+菽
+菿
+萁
+萃
+萄
+萆
+萋
+萌
+萍
+萎
+萏
+萑
+萘
+萜
+萝
+萢
+萤
+营
+萦
+萧
+萨
+萩
+萱
+萸
+萼
+落
+葆
+葎
+葑
+葖
+著
+葙
+葚
+葛
+葜
+葡
+葢
+董
+葩
+葫
+葬
+葭
+葱
+葳
+葵
+葶
+葸
+葺
+蒂
+蒇
+蒉
+蒋
+蒌
+蒍
+蒎
+蒐
+蒗
+蒙
+蒜
+蒟
+蒡
+蒨
+蒯
+蒲
+蒴
+蒸
+蒹
+蒺
+蒻
+蒽
+蒾
+蒿
+蓁
+蓂
+蓄
+蓇
+蓉
+蓊
+蓍
+蓐
+蓑
+蓓
+蓖
+蓝
+蓟
+蓠
+蓢
+蓣
+蓥
+蓦
+蓬
+蓼
+蓿
+蔀
+蔌
+蔑
+蔓
+蔗
+蔘
+蔚
+蔟
+蔡
+蔫
+蔬
+蔴
+蔵
+蔷
+蔸
+蔹
+蔺
+蔻
+蔼
+蔽
+蕃
+蕅
+蕈
+蕉
+蕊
+蕖
+蕗
+蕙
+蕞
+蕡
+蕤
+蕨
+蕰
+蕲
+蕴
+蕹
+蕺
+蕻
+蕾
+薄
+薅
+薆
+薇
+薏
+薙
+薛
+薜
+薢
+薤
+薨
+薪
+薫
+薬
+薮
+薯
+薰
+薷
+薹
+藁
+藉
+藏
+藐
+藓
+藕
+藜
+藟
+藠
+藤
+藦
+藨
+藩
+藻
+藿
+蘅
+蘑
+蘖
+蘧
+蘩
+蘸
+蘼
+虎
+虏
+虐
+虑
+虒
+虓
+虔
+虚
+虞
+虢
+虫
+虬
+虮
+虱
+虹
+虺
+虻
+虽
+虾
+虿
+蚀
+蚁
+蚂
+蚊
+蚋
+蚌
+蚍
+蚓
+蚕
+蚖
+蚜
+蚝
+蚡
+蚣
+蚤
+蚧
+蚨
+蚩
+蚪
+蚬
+蚯
+蚰
+蚱
+蚴
+蚵
+蚶
+蚺
+蛀
+蛄
+蛆
+蛇
+蛉
+蛊
+蛋
+蛎
+蛏
+蛐
+蛔
+蛙
+蛛
+蛞
+蛟
+蛤
+蛩
+蛭
+蛮
+蛰
+蛱
+蛲
+蛳
+蛴
+蛸
+蛹
+蛾
+蜀
+蜂
+蜃
+蜇
+蜈
+蜉
+蜊
+蜍
+蜑
+蜒
+蜓
+蜕
+蜗
+蜘
+蜚
+蜜
+蜞
+蜡
+蜢
+蜣
+蜥
+蜩
+蜮
+蜱
+蜴
+蜷
+蜻
+蜾
+蜿
+蝇
+蝈
+蝉
+蝌
+蝎
+蝓
+蝗
+蝙
+蝠
+蝣
+蝤
+蝥
+蝮
+蝰
+蝲
+蝴
+蝶
+蝻
+蝼
+蝽
+蝾
+螂
+螃
+螅
+螈
+螋
+融
+螓
+螟
+螣
+螨
+螫
+螬
+螭
+螯
+螳
+螵
+螺
+螽
+蟀
+蟆
+蟊
+蟋
+蟌
+蟑
+蟒
+蟛
+蟜
+蟠
+蟥
+蟪
+蟮
+蟳
+蟹
+蟾
+蠃
+蠊
+蠋
+蠍
+蠓
+蠕
+蠖
+蠡
+蠢
+蠲
+蠹
+蠼
+血
+衄
+衅
+衆
+行
+衍
+衎
+衔
+街
+衙
+衞
+衡
+衢
+衣
+补
+表
+衩
+衪
+衫
+衬
+衮
+衰
+衲
+衷
+衽
+衾
+衿
+袁
+袂
+袄
+袅
+袆
+袈
+袋
+袍
+袒
+袓
+袖
+袛
+袜
+袢
+袤
+袪
+被
+袭
+袮
+袱
+袴
+袷
+袼
+裁
+裂
+装
+裆
+裈
+裉
+裒
+裔
+裕
+裘
+裙
+裛
+裟
+裡
+裢
+裤
+裥
+裨
+裪
+裱
+裳
+裴
+裸
+裹
+裼
+裾
+褀
+褂
+褆
+褊
+褐
+褒
+褓
+褔
+褙
+褚
+褛
+褡
+褥
+褪
+褫
+褰
+褴
+褶
+襁
+襄
+襌
+襕
+襜
+襞
+襟
+襦
+襻
+西
+要
+覃
+覆
+覇
+覚
+覧
+覩
+観
+见
+观
+规
+觅
+视
+觇
+览
+觉
+觊
+觋
+觌
+觎
+觏
+觐
+觑
+角
+觚
+觜
+觞
+解
+觥
+触
+觯
+觳
+觽
+觿
+言
+訇
+訏
+訚
+訫
+訳
+訾
+詈
+詜
+詝
+詧
+詹
+誉
+誊
+誐
+誓
+説
+読
+諡
+諲
+諴
+謇
+謦
+譞
+警
+譬
+譲
+讌
+讠
+计
+订
+讣
+认
+讥
+讦
+讧
+讨
+让
+讪
+讫
+讬
+训
+议
+讯
+记
+讲
+讳
+讴
+讵
+讶
+讷
+许
+讹
+论
+讼
+讽
+设
+访
+诀
+证
+诂
+诃
+评
+诅
+识
+诈
+诉
+诊
+诋
+诌
+词
+诎
+诏
+译
+诒
+诓
+诔
+试
+诖
+诗
+诘
+诙
+诚
+诛
+诜
+话
+诞
+诟
+诠
+诡
+询
+诣
+诤
+该
+详
+诧
+诨
+诩
+诫
+诬
+语
+诮
+误
+诰
+诱
+诲
+诳
+说
+诵
+诶
+请
+诸
+诹
+诺
+读
+诼
+诽
+课
+诿
+谀
+谁
+谂
+调
+谄
+谅
+谆
+谇
+谈
+谊
+谋
+谌
+谍
+谎
+谏
+谐
+谑
+谒
+谓
+谔
+谕
+谖
+谗
+谘
+谙
+谚
+谛
+谜
+谝
+谞
+谟
+谠
+谡
+谢
+谣
+谤
+谥
+谦
+谧
+谨
+谩
+谪
+谫
+谬
+谭
+谮
+谯
+谰
+谱
+谲
+谳
+谴
+谵
+谶
+谷
+谿
+豁
+豆
+豇
+豉
+豊
+豌
+豕
+豚
+象
+豢
+豨
+豪
+豫
+豳
+豸
+豹
+豺
+貂
+貅
+貉
+貊
+貌
+貐
+貔
+貘
+賨
+賸
+贇
+贝
+贞
+负
+贠
+贡
+财
+责
+贤
+败
+账
+货
+质
+贩
+贪
+贫
+贬
+购
+贮
+贯
+贰
+贱
+贲
+贳
+贴
+贵
+贶
+贷
+贸
+费
+贺
+贻
+贼
+贽
+贾
+贿
+赀
+赁
+赂
+赃
+资
+赅
+赈
+赉
+赊
+赋
+赌
+赍
+赎
+赏
+赐
+赑
+赓
+赔
+赕
+赖
+赘
+赙
+赚
+赛
+赜
+赝
+赞
+赟
+赠
+赡
+赢
+赣
+赤
+赦
+赧
+赪
+赫
+赭
+赮
+走
+赳
+赴
+赵
+赶
+起
+趁
+趄
+超
+越
+趋
+趔
+趟
+趣
+趯
+趱
+足
+趴
+趵
+趸
+趺
+趼
+趾
+趿
+跂
+跃
+跄
+跆
+跋
+跌
+跎
+跏
+跑
+跖
+跗
+跚
+跛
+距
+跞
+跟
+跢
+跣
+跤
+跨
+跩
+跪
+跫
+跬
+路
+跳
+践
+跶
+跷
+跸
+跹
+跺
+跻
+跽
+踅
+踉
+踊
+踌
+踏
+踔
+踝
+踞
+踟
+踢
+踣
+踩
+踪
+踬
+踮
+踯
+踰
+踱
+踵
+踹
+踽
+蹀
+蹁
+蹂
+蹄
+蹇
+蹈
+蹉
+蹊
+蹋
+蹑
+蹒
+蹓
+蹙
+蹚
+蹟
+蹦
+蹩
+蹬
+蹭
+蹰
+蹲
+蹴
+蹶
+蹻
+蹼
+蹿
+躁
+躄
+躅
+躇
+躏
+躐
+躔
+躜
+躞
+身
+躬
+躯
+躲
+躺
+転
+軽
+輋
+轘
+车
+轧
+轨
+轩
+轫
+转
+轭
+轮
+软
+轰
+轱
+轲
+轳
+轴
+轵
+轶
+轸
+轹
+轺
+轻
+轼
+载
+轾
+轿
+辂
+较
+辄
+辅
+辆
+辇
+辈
+辉
+辊
+辋
+辍
+辎
+辏
+辐
+辑
+输
+辔
+辕
+辖
+辗
+辘
+辙
+辚
+辛
+辜
+辞
+辟
+辣
+辨
+辩
+辫
+辰
+辱
+辶
+边
+辺
+辻
+込
+辽
+达
+辿
+迁
+迂
+迄
+迅
+过
+迈
+迍
+迎
+运
+近
+迓
+返
+迕
+还
+这
+进
+远
+违
+连
+迟
+迢
+迤
+迥
+迦
+迨
+迩
+迪
+迫
+迭
+迮
+述
+迳
+迷
+迸
+迹
+追
+退
+送
+适
+逃
+逄
+逅
+逆
+选
+逊
+逋
+逍
+透
+逐
+逑
+递
+途
+逖
+逗
+這
+通
+逛
+逝
+逞
+速
+造
+逡
+逢
+逦
+逭
+逮
+逯
+進
+逵
+逶
+逸
+逹
+逺
+逻
+逼
+逾
+遁
+遂
+遄
+遇
+遍
+遏
+遐
+遑
+遒
+道
+遗
+遘
+遛
+遢
+遣
+遥
+遨
+遭
+遮
+遯
+遴
+遵
+遶
+遹
+遽
+避
+邀
+邂
+邃
+邅
+邈
+邉
+邋
+邑
+邓
+邕
+邗
+邙
+邛
+邝
+邠
+邡
+邢
+那
+邦
+邨
+邪
+邬
+邮
+邯
+邰
+邱
+邳
+邴
+邵
+邶
+邸
+邹
+邺
+邻
+邽
+邾
+郁
+郄
+郅
+郇
+郈
+郊
+郎
+郏
+郐
+郑
+郓
+郕
+郗
+郚
+郛
+郜
+郝
+郞
+郡
+郢
+郤
+郦
+郧
+部
+郪
+郫
+郭
+郯
+郴
+郷
+郸
+都
+郾
+郿
+鄀
+鄂
+鄄
+鄗
+鄘
+鄙
+鄚
+鄜
+鄞
+鄠
+鄢
+鄣
+鄩
+鄫
+鄮
+鄯
+鄱
+鄹
+酂
+酃
+酆
+酉
+酊
+酋
+酌
+配
+酎
+酐
+酒
+酗
+酚
+酝
+酞
+酡
+酢
+酣
+酤
+酥
+酩
+酪
+酬
+酮
+酯
+酰
+酱
+酲
+酴
+酵
+酶
+酷
+酸
+酹
+酺
+酽
+酾
+酿
+醂
+醅
+醇
+醉
+醋
+醌
+醍
+醐
+醑
+醒
+醚
+醛
+醢
+醣
+醪
+醭
+醮
+醯
+醴
+醵
+醺
+醿
+釆
+采
+釉
+释
+里
+重
+野
+量
+金
+釜
+釭
+釿
+鈇
+鈈
+鈊
+鈎
+鈡
+鉄
+鉏
+鉨
+鉴
+鉷
+銎
+銙
+銭
+銮
+鋆
+鋈
+鋐
+鋗
+鋬
+鋹
+錞
+錡
+錤
+録
+錾
+鍀
+鍪
+鎌
+鎏
+鎚
+鏊
+鏐
+鏖
+鐏
+鑨
+鑫
+钅
+钆
+钇
+针
+钉
+钊
+钋
+钌
+钍
+钎
+钏
+钐
+钒
+钓
+钔
+钕
+钖
+钗
+钙
+钚
+钛
+钜
+钝
+钞
+钟
+钠
+钡
+钢
+钣
+钤
+钥
+钦
+钧
+钨
+钩
+钪
+钫
+钬
+钭
+钮
+钯
+钰
+钱
+钲
+钳
+钴
+钵
+钶
+钷
+钹
+钺
+钻
+钼
+钽
+钾
+钿
+铀
+铁
+铂
+铃
+铄
+铅
+铆
+铈
+铉
+铊
+铋
+铌
+铍
+铎
+铐
+铑
+铒
+铓
+铕
+铖
+铗
+铙
+铚
+铛
+铜
+铝
+铟
+铠
+铡
+铢
+铣
+铤
+铥
+铦
+铧
+铨
+铩
+铪
+铫
+铬
+铭
+铮
+铯
+铰
+铱
+铲
+铳
+铵
+银
+铷
+铸
+铺
+铼
+铽
+链
+铿
+销
+锁
+锂
+锃
+锄
+锅
+锆
+锇
+锈
+锉
+锋
+锌
+锍
+锎
+锏
+锐
+锑
+锒
+锔
+锕
+锖
+锗
+锘
+错
+锚
+锛
+锜
+锝
+锞
+锟
+锠
+锡
+锢
+锣
+锤
+锥
+锦
+锨
+锩
+锪
+锫
+锬
+锭
+键
+锯
+锰
+锱
+锲
+锳
+锴
+锵
+锶
+锷
+锸
+锹
+锺
+锻
+锼
+锽
+镀
+镁
+镂
+镄
+镅
+镆
+镇
+镈
+镉
+镊
+镋
+镌
+镍
+镎
+镏
+镐
+镑
+镒
+镓
+镔
+镕
+镖
+镗
+镘
+镙
+镚
+镛
+镜
+镝
+镞
+镟
+镠
+镡
+镢
+镣
+镥
+镦
+镧
+镨
+镪
+镫
+镬
+镭
+镯
+镰
+镱
+镲
+镳
+镶
+长
+開
+閟
+関
+閦
+闇
+闍
+闘
+门
+闩
+闪
+闫
+闭
+问
+闯
+闰
+闱
+闲
+闳
+间
+闵
+闶
+闷
+闸
+闹
+闺
+闻
+闼
+闽
+闾
+闿
+阀
+阁
+阂
+阃
+阄
+阅
+阆
+阇
+阈
+阉
+阊
+阋
+阌
+阍
+阎
+阏
+阐
+阑
+阒
+阓
+阔
+阕
+阖
+阗
+阙
+阚
+阛
+阜
+阝
+队
+阡
+阪
+阮
+阱
+防
+阳
+阴
+阵
+阶
+阻
+阼
+阿
+陀
+陂
+附
+际
+陆
+陇
+陈
+陉
+陋
+陌
+降
+限
+陔
+陕
+陛
+陞
+陟
+陡
+院
+除
+陨
+险
+陪
+陬
+陲
+陵
+陶
+陷
+険
+隂
+隅
+隆
+隈
+隋
+隍
+随
+隐
+隔
+隗
+隘
+隙
+障
+隠
+隣
+隧
+隰
+隳
+隶
+隹
+隼
+隽
+难
+雀
+雁
+雄
+雅
+集
+雇
+雉
+雊
+雌
+雍
+雎
+雏
+雑
+雒
+雕
+雠
+雨
+雩
+雪
+雫
+雯
+雱
+雳
+零
+雷
+雹
+雾
+需
+霁
+霂
+霄
+霅
+霆
+震
+霈
+霉
+霊
+霍
+霎
+霏
+霓
+霖
+霙
+霜
+霞
+霪
+霭
+霰
+露
+霸
+霹
+霾
+靑
+青
+靓
+靖
+静
+靛
+非
+靠
+靡
+面
+靥
+革
+靬
+靳
+靴
+靶
+靺
+靼
+鞅
+鞋
+鞍
+鞑
+鞔
+鞘
+鞞
+鞠
+鞣
+鞥
+鞨
+鞫
+鞬
+鞭
+鞮
+鞯
+鞴
+韘
+韡
+韦
+韧
+韩
+韪
+韫
+韬
+韭
+音
+韵
+韶
+頔
+頞
+頠
+頫
+頵
+頼
+顒
+顔
+顕
+顗
+页
+顶
+顷
+顸
+项
+顺
+须
+顼
+顽
+顾
+顿
+颀
+颁
+颂
+颃
+预
+颅
+领
+颇
+颈
+颉
+颊
+颋
+颌
+颍
+颎
+颏
+颐
+频
+颓
+颔
+颖
+颗
+题
+颙
+颚
+颛
+颜
+额
+颞
+颟
+颠
+颡
+颢
+颤
+颦
+颧
+风
+飏
+飐
+飑
+飒
+飓
+飕
+飖
+飘
+飙
+飚
+飞
+食
+飧
+飨
+餍
+餐
+餗
+餮
+饔
+饕
+饣
+饥
+饦
+饧
+饨
+饩
+饪
+饫
+饬
+饭
+饮
+饯
+饰
+饱
+饲
+饴
+饵
+饶
+饷
+饸
+饹
+饺
+饼
+饽
+饿
+馀
+馁
+馃
+馄
+馅
+馆
+馇
+馈
+馊
+馋
+馍
+馏
+馐
+馑
+馒
+馓
+馔
+馕
+首
+馗
+馘
+香
+馥
+馨
+駄
+駅
+駆
+騄
+騑
+騒
+験
+驎
+驒
+驩
+马
+驭
+驮
+驯
+驰
+驱
+驳
+驴
+驶
+驷
+驸
+驹
+驺
+驻
+驼
+驽
+驾
+驿
+骀
+骁
+骂
+骃
+骄
+骅
+骆
+骇
+骈
+骉
+骊
+骋
+验
+骍
+骎
+骏
+骐
+骑
+骓
+骕
+骖
+骗
+骘
+骙
+骚
+骛
+骜
+骝
+骞
+骟
+骠
+骡
+骢
+骤
+骥
+骧
+骨
+骰
+骱
+骶
+骷
+骸
+骺
+骼
+髀
+髁
+髂
+髃
+髅
+髈
+髋
+髌
+髎
+髑
+髓
+高
+髙
+髡
+髦
+髪
+髫
+髭
+髯
+髹
+髻
+鬃
+鬈
+鬐
+鬓
+鬘
+鬟
+鬣
+鬯
+鬲
+鬶
+鬻
+鬼
+魁
+魂
+魃
+魄
+魅
+魆
+魇
+魈
+魉
+魋
+魍
+魏
+魑
+魔
+魟
+鮀
+鮈
+鮋
+鮟
+鮠
+鮨
+鮰
+鰕
+鰤
+鱀
+鱇
+鱓
+鱬
+鱲
+鱻
+鱼
+鱿
+鲀
+鲁
+鲂
+鲃
+鲅
+鲆
+鲇
+鲈
+鲉
+鲊
+鲋
+鲌
+鲍
+鲎
+鲏
+鲐
+鲑
+鲔
+鲕
+鲗
+鲘
+鲙
+鲚
+鲛
+鲜
+鲞
+鲟
+鲠
+鲡
+鲢
+鲣
+鲤
+鲥
+鲦
+鲧
+鲨
+鲩
+鲫
+鲭
+鲮
+鲱
+鲲
+鲳
+鲴
+鲵
+鲶
+鲷
+鲸
+鲹
+鲺
+鲻
+鲼
+鲽
+鲿
+鳀
+鳃
+鳄
+鳅
+鳇
+鳉
+鳊
+鳌
+鳍
+鳎
+鳏
+鳐
+鳑
+鳓
+鳔
+鳕
+鳖
+鳗
+鳙
+鳚
+鳜
+鳝
+鳞
+鳟
+鳡
+鳢
+鳣
+鳯
+鳽
+鳾
+鴂
+鴞
+鴷
+鵖
+鵙
+鵟
+鵺
+鶒
+鶲
+鷇
+鷉
+鷟
+鸂
+鸊
+鸑
+鸟
+鸠
+鸡
+鸢
+鸣
+鸥
+鸦
+鸨
+鸩
+鸪
+鸫
+鸬
+鸭
+鸮
+鸯
+鸰
+鸱
+鸲
+鸳
+鸵
+鸶
+鸷
+鸸
+鸹
+鸺
+鸻
+鸽
+鸾
+鸿
+鹀
+鹁
+鹂
+鹃
+鹄
+鹅
+鹆
+鹇
+鹈
+鹉
+鹊
+鹋
+鹌
+鹍
+鹎
+鹏
+鹑
+鹓
+鹕
+鹖
+鹗
+鹘
+鹚
+鹛
+鹜
+鹞
+鹟
+鹠
+鹡
+鹢
+鹣
+鹤
+鹦
+鹧
+鹨
+鹩
+鹪
+鹫
+鹬
+鹭
+鹮
+鹰
+鹱
+鹳
+鹾
+鹿
+麂
+麇
+麈
+麋
+麐
+麑
+麒
+麓
+麝
+麟
+麤
+麦
+麴
+麸
+麹
+麻
+麽
+麾
+麿
+黄
+黉
+黍
+黎
+黏
+黐
+黑
+黒
+黔
+默
+黙
+黛
+黜
+黝
+黟
+黠
+黡
+黢
+黥
+黧
+黩
+黯
+黻
+黼
+黾
+鼋
+鼍
+鼎
+鼐
+鼓
+鼗
+鼙
+鼠
+鼢
+鼩
+鼬
+鼯
+鼱
+鼷
+鼹
+鼻
+鼽
+鼾
+齁
+齐
+齑
+齢
+齮
+齿
+龀
+龁
+龃
+龄
+龅
+龆
+龇
+龈
+龉
+龊
+龋
+龌
+龑
+龘
+龙
+龚
+龛
+龟
+龠
+龢
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+AA
+AB
+AC
+AD
+AE
+AF
+AG
+AH
+AI
+AJ
+AK
+AL
+AM
+AN
+AP
+AQ
+AR
+AS
+AT
+AU
+AV
+AW
+AX
+AZ
+Al
+An
+Au
+Aw
+BA
+BB
+BC
+BD
+BE
+BF
+BG
+BH
+BI
+BJ
+BK
+BL
+BM
+BN
+BO
+BP
+BQ
+BR
+BS
+BT
+BU
+BV
+BW
+BY
+Bo
+Br
+Bu
+CA
+CB
+CC
+CD
+CE
+CF
+CG
+CH
+CI
+CJ
+CK
+CL
+CM
+CN
+CO
+CP
+CQ
+CR
+CS
+CT
+CU
+CV
+CW
+CX
+CY
+CZ
+Ca
+Ch
+Cl
+Co
+Cu
+DA
+DB
+DC
+DD
+DE
+DF
+DG
+DH
+DI
+DJ
+DK
+DL
+DM
+DN
+DO
+DQ
+DR
+DS
+DT
+DV
+DW
+DX
+DY
+DZ
+Da
+De
+Di
+Do
+Dr
+Du
+EA
+EB
+EC
+ED
+EE
+EF
+EG
+EH
+EI
+EK
+EL
+EM
+EN
+EP
+EQ
+ER
+ES
+ET
+EU
+EV
+EW
+EX
+EZ
+Ed
+En
+Ev
+Ex
+FA
+FB
+FC
+FD
+FE
+FF
+FG
+FH
+FI
+FJ
+FL
+FM
+FN
+FO
+FP
+FR
+FS
+FT
+FU
+FW
+FX
+FY
+FZ
+Fa
+Fi
+Fl
+Fo
+Fr
+Fu
+GA
+GB
+GC
+GD
+GE
+GF
+GG
+GH
+GI
+GJ
+GK
+GL
+GM
+GN
+GO
+GP
+GQ
+GR
+GS
+GT
+GU
+GW
+GX
+GY
+GZ
+Ga
+Go
+Gr
+Gu
+HA
+HB
+HC
+HD
+HE
+HF
+HG
+HH
+HI
+HJ
+HK
+HL
+HO
+HP
+HQ
+HR
+HS
+HT
+HU
+HV
+HW
+HX
+HY
+HZ
+Ha
+He
+Hi
+Ho
+Hu
+Hz
+IB
+IC
+ID
+IE
+IF
+IG
+IH
+II
+IK
+IL
+IM
+IN
+IO
+IP
+IQ
+IR
+IS
+IT
+IU
+IV
+IX
+If
+In
+JA
+JB
+JC
+JD
+JF
+JG
+JH
+JI
+JJ
+JK
+JL
+JM
+JO
+JP
+JQ
+JR
+JS
+JT
+JU
+JW
+JX
+JY
+JZ
+Ja
+Ji
+Jo
+Ju
+KA
+KB
+KC
+KD
+KE
+KF
+KG
+KH
+KI
+KJ
+KK
+KL
+KM
+KN
+KO
+KP
+KR
+KS
+KT
+KV
+KW
+KX
+KY
+KZ
+LA
+LB
+LC
+LD
+LE
+LF
+LG
+LH
+LI
+LJ
+LK
+LL
+LM
+LN
+LO
+LP
+LQ
+LR
+LS
+LT
+LU
+LV
+LW
+LX
+LY
+LZ
+La
+Le
+Li
+Lo
+Lu
+MA
+MB
+MC
+MD
+ME
+MF
+MG
+MH
+MI
+MJ
+MK
+ML
+MM
+MN
+MO
+MP
+MQ
+MR
+MS
+MT
+MU
+MV
+MW
+MX
+MY
+Ma
+Me
+Mi
+Mo
+Mu
+My
+NA
+NB
+NC
+ND
+NE
+NF
+NG
+NH
+NI
+NJ
+NK
+NL
+NN
+NO
+NP
+NR
+NS
+NT
+NU
+NV
+NW
+NX
+NY
+NZ
+Na
+Ne
+No
+Nu
+OA
+OB
+OC
+OD
+OE
+OF
+OG
+OH
+OK
+OL
+OM
+ON
+OO
+OP
+OR
+OS
+OT
+OU
+OV
+OZ
+Of
+Oh
+On
+Op
+Or
+Ou
+Ox
+PA
+PB
+PC
+PD
+PE
+PF
+PG
+PH
+PI
+PJ
+PK
+PL
+PM
+PN
+PO
+PP
+PQ
+PR
+PS
+PT
+PU
+PV
+PW
+PX
+Pa
+Ph
+Pl
+Po
+Pr
+Pu
+QA
+QB
+QC
+QE
+QF
+QG
+QJ
+QL
+QQ
+QR
+QS
+QT
+QU
+QW
+QY
+Qi
+Qu
+RA
+RB
+RC
+RE
+RF
+RG
+RH
+RI
+RJ
+RK
+RL
+RM
+RN
+RO
+RP
+RQ
+RR
+RS
+RT
+RV
+RW
+RX
+RZ
+Ra
+Re
+Ro
+Ru
+SA
+SB
+SC
+SD
+SE
+SF
+SG
+SH
+SI
+SJ
+SK
+SL
+SM
+SN
+SO
+SP
+SQ
+SR
+SS
+ST
+SU
+SV
+SW
+SX
+SY
+SZ
+Sc
+Sh
+So
+Sp
+St
+Su
+Sw
+Sy
+TA
+TB
+TC
+TD
+TE
+TF
+TG
+TH
+TI
+TJ
+TK
+TL
+TM
+TN
+TO
+TP
+TQ
+TR
+TS
+TT
+TU
+TV
+TW
+TX
+TY
+TZ
+Th
+To
+Tr
+Tw
+UA
+UC
+UD
+UE
+UF
+UG
+UH
+UI
+UK
+UL
+UM
+UN
+UP
+UR
+US
+UT
+UU
+UV
+UW
+UX
+Ub
+Un
+Up
+VA
+VB
+VC
+VE
+VF
+VG
+VH
+VI
+VJ
+VK
+VL
+VM
+VN
+VO
+VP
+VR
+VS
+VT
+VU
+VV
+VX
+Vi
+Vo
+WA
+WB
+WC
+WE
+WH
+WI
+WJ
+WL
+WM
+WN
+WO
+WQ
+WR
+WS
+WT
+WU
+WW
+WX
+WZ
+Wa
+We
+Wi
+Wo
+Wu
+XB
+XC
+XD
+XF
+XG
+XH
+XI
+XJ
+XK
+XL
+XM
+XO
+XP
+XQ
+XR
+XS
+XT
+XU
+XV
+XW
+XX
+XY
+XZ
+Xi
+Xu
+YA
+YB
+YC
+YD
+YE
+YF
+YG
+YH
+YJ
+YL
+YM
+YO
+YP
+YS
+YT
+YU
+YX
+YY
+YZ
+Ya
+Yo
+Yu
+ZA
+ZB
+ZC
+ZD
+ZE
+ZF
+ZG
+ZH
+ZI
+ZJ
+ZL
+ZM
+ZN
+ZO
+ZQ
+ZR
+ZS
+ZU
+ZW
+ZX
+ZY
+ZZ
+Zh
+ab
+aj
+an
+ap
+ar
+bb
+be
+bj
+bo
+bu
+by
+ca
+cb
+cf
+ch
+cl
+cm
+co
+cp
+cv
+dB
+da
+de
+di
+dj
+dn
+do
+dr
+dv
+ed
+em
+en
+ep
+eq
+ev
+ex
+ez
+fa
+fe
+ff
+fi
+fl
+fo
+fr
+fu
+gb
+gd
+gh
+gi
+go
+gp
+gr
+gu
+gz
+ha
+he
+hi
+ho
+hp
+hz
+iP
+iT
+ib
+ic
+id
+if
+ig
+im
+in
+io
+ip
+iq
+is
+it
+jQ
+ja
+ji
+jj
+jo
+jq
+ju
+kJ
+kN
+kW
+kg
+kn
+kz
+la
+ld
+le
+lg
+li
+ll
+lo
+lp
+lz
+ma
+mb
+me
+mi
+mm
+mo
+mp
+mq
+mu
+mv
+my
+na
+nb
+ng
+no
+nv
+ob
+of
+oh
+ok
+ol
+on
+op
+or
+ou
+ow
+oz
+pH
+pa
+pc
+ph
+pk
+pl
+po
+pp
+pr
+pu
+pv
+qf
+qq
+qu
+qz
+ra
+re
+rn
+ro
+rq
+se
+sh
+sk
+so
+sp
+sq
+st
+su
+sw
+sz
+th
+ti
+to
+tr
+tv
+tw
+ub
+uc
+uf
+ui
+uk
+un
+up
+us
+uv
+ux
+uz
+vc
+vi
+vo
+vr
+wa
+we
+wh
+wi
+wo
+wr
+ww
+xj
+xq
+xx
+ya
+ye
+yj
+yo
+yu
+yy
+yz
+zf
+zh
+zi
+zj
+zq
+zu
+zz
+AAA
+AAC
+ABA
+ABB
+ABC
+ABO
+ABS
+ABT
+ACA
+ACC
+ACD
+ACE
+ACG
+ACK
+ACL
+ACM
+ACP
+ACR
+ACS
+ACT
+ADA
+ADC
+ADD
+ADF
+ADI
+ADO
+ADP
+ADR
+ADS
+ADV
+AED
+AES
+AFC
+AFP
+AFS
+AGB
+AGC
+AGE
+AGM
+AGP
+AGV
+AIA
+AIC
+AIG
+AIM
+AIP
+AIR
+AIS
+AIX
+AKB
+AKM
+ALA
+ALL
+ALT
+AMA
+AMC
+AMD
+AMG
+AMI
+AML
+AMP
+AMR
+AMS
+AMT
+AMX
+AND
+AOC
+AOE
+AOL
+APA
+APC
+APE
+APG
+API
+APK
+APL
+APM
+APP
+APS
+APT
+APU
+ARA
+ARC
+ARE
+ARM
+ARP
+ART
+ASA
+ASC
+ASF
+ASM
+ASP
+ASR
+AST
+ATA
+ATC
+ATF
+ATI
+ATK
+ATM
+ATP
+ATS
+ATV
+ATX
+AUC
+AUG
+AUX
+AVC
+AVG
+AVI
+AVR
+AVS
+AVX
+AWM
+AWS
+All
+And
+Ang
+App
+Aqu
+BAC
+BAD
+BAE
+BAR
+BAT
+BAU
+BBA
+BBB
+BBC
+BBE
+BBQ
+BBS
+BBT
+BCD
+BEA
+BEC
+BEI
+BET
+BGA
+BGM
+BGP
+BIG
+BIM
+BIS
+BLG
+BMC
+BMD
+BMG
+BMI
+BMP
+BMW
+BMX
+BNC
+BOD
+BOM
+BOT
+BOX
+BOY
+BPM
+BPO
+BRN
+BRT
+BSA
+BSC
+BSD
+BSI
+BSM
+BSP
+BSS
+BTC
+BTR
+BTS
+BTV
+BUG
+BUN
+BUS
+BWV
+Bur
+Bus
+But
+CAA
+CAC
+CAD
+CAE
+CAI
+CAJ
+CAM
+CAN
+CAP
+CAR
+CAS
+CAT
+CBA
+CBC
+CBD
+CBN
+CBR
+CBS
+CCA
+CCC
+CCD
+CCF
+CCG
+CCI
+CCK
+CCM
+CCN
+CCP
+CCS
+CDC
+CDM
+CDN
+CDO
+CDP
+CDR
+CDS
+CEA
+CEC
+CEO
+CES
+CET
+CFA
+CFC
+CFD
+CFO
+CFR
+CGI
+CHA
+CHM
+CHO
+CIA
+CIC
+CID
+CIE
+CIF
+CIK
+CIO
+CIP
+CIS
+CLA
+CLI
+CLM
+CLS
+CMA
+CMC
+CME
+CML
+CMM
+CMO
+CMP
+CMS
+CMV
+CNC
+CNG
+CNN
+CNS
+COB
+COC
+COD
+COM
+CON
+COO
+COP
+COS
+COX
+CPA
+CPC
+CPE
+CPI
+CPL
+CPM
+CPP
+CPR
+CPS
+CPU
+CQC
+CRC
+CRM
+CRP
+CRS
+CRT
+CSA
+CSF
+CSI
+CSM
+CSP
+CSR
+CSS
+CST
+CTA
+CTC
+CTI
+CTO
+CTP
+CTS
+CUB
+CUT
+CVD
+CVN
+CVS
+CVT
+CXW
+CYP
+Car
+Cha
+Chr
+Chu
+Com
+Con
+Cou
+Cur
+DAB
+DAC
+DAO
+DAS
+DAT
+DAY
+DBA
+DBM
+DCD
+DCE
+DCF
+DCS
+DCT
+DDC
+DDD
+DDG
+DDN
+DDR
+DDS
+DDT
+DEA
+DEC
+DEM
+DES
+DFS
+DFT
+DHA
+DHL
+DIC
+DID
+DIF
+DIN
+DIP
+DIV
+DIY
+DLC
+DLL
+DLP
+DLT
+DMA
+DMC
+DMD
+DMF
+DMI
+DMO
+DMZ
+DNA
+DNF
+DNS
+DNV
+DOC
+DOI
+DOM
+DON
+DOS
+DOT
+DPI
+DPP
+DPS
+DRM
+DRX
+DSA
+DSC
+DSG
+DSL
+DSM
+DSP
+DSS
+DTC
+DTE
+DTM
+DTS
+DTU
+DVB
+DVD
+DVI
+DVR
+DWG
+DYG
+Day
+Div
+Don
+Dou
+Dow
+EAN
+EAP
+EBD
+EBS
+ECC
+ECM
+ECO
+ECT
+ECU
+ECW
+EDA
+EDG
+EDI
+EDM
+EDP
+EDR
+EEG
+EEP
+EFR
+EGF
+EHS
+EIA
+EJB
+EMA
+EMC
+EMI
+EMP
+EMS
+END
+EOS
+EPA
+EPC
+EPO
+EPR
+EPS
+ERP
+ESC
+ESD
+ESI
+ESL
+ESP
+ESR
+EST
+ETC
+ETF
+ETH
+ETL
+ETS
+EVA
+EVE
+EVO
+EXE
+EXO
+EXP
+EYE
+Eff
+Ell
+Emb
+Emp
+End
+Eng
+Equ
+Eur
+Eva
+Exc
+Exp
+FAA
+FAB
+FAG
+FAL
+FAN
+FAO
+FAQ
+FAT
+FBI
+FCA
+FCC
+FCI
+FCS
+FDA
+FDD
+FDI
+FEM
+FES
+FET
+FFT
+FGO
+FHD
+FIA
+FLV
+FLY
+FMS
+FNC
+FOB
+FOF
+FOR
+FOX
+FPC
+FPS
+FPX
+FRP
+FSA
+FSB
+FSC
+FSH
+FTA
+FTC
+FTP
+FUE
+FUN
+Fin
+Fiv
+Fly
+For
+Fou
+Fuj
+Fun
+Fut
+GAP
+GAT
+GAY
+GBA
+GBK
+GBT
+GBU
+GCC
+GCS
+GCT
+GDI
+GDP
+GEN
+GEO
+GET
+GFP
+GHz
+GIA
+GIF
+GIS
+GLA
+GLC
+GLP
+GLS
+GMA
+GMC
+GMP
+GMS
+GMT
+GMV
+GND
+GNP
+GNU
+GOD
+GOT
+GPA
+GPL
+GPS
+GPT
+GPU
+GRC
+GRE
+GRF
+GSH
+GSM
+GSP
+GTA
+GTI
+GTO
+GTP
+GTR
+GTS
+GTX
+GUI
+Giv
+Gmb
+Gua
+Gui
+Gun
+Guo
+Guy
+HAD
+HAL
+HBA
+HBO
+HBV
+HBs
+HCG
+HCI
+HCV
+HCl
+HDD
+HDL
+HDR
+HDV
+HEY
+HFC
+HGH
+HGT
+HID
+HIP
+HIS
+HIT
+HIV
+HLA
+HMG
+HMI
+HMS
+HOP
+HOT
+HOW
+HPC
+HPV
+HRC
+HRT
+HSE
+HSK
+HSV
+HTC
+HUB
+HUD
+HVG
+Haz
+Her
+Hom
+Hon
+Hou
+How
+Hua
+Hub
+Hum
+Hun
+IAI
+IAS
+IAT
+IBC
+IBF
+IBM
+ICA
+ICC
+ICD
+ICE
+ICO
+ICP
+ICQ
+ICS
+ICT
+ICU
+IDC
+IDD
+IDE
+IDF
+IDG
+IDS
+IEC
+IET
+IFA
+IFC
+IFI
+IFN
+IGF
+IGN
+IIA
+III
+IIS
+IKO
+IMA
+IMC
+IMD
+IME
+IMF
+IMG
+IMO
+IMS
+IMT
+INA
+INC
+INF
+ING
+INS
+INT
+IOS
+IPA
+IPC
+IPO
+IPS
+IPX
+IRC
+IRI
+ISA
+ISI
+ISM
+ISO
+ISP
+ITC
+ITF
+ITO
+ITS
+ITT
+ITU
+ITV
+IVR
+Imp
+InC
+Inf
+Inj
+Int
+JAR
+JBL
+JBT
+JCB
+JCR
+JDB
+JDG
+JET
+JGJ
+JIS
+JIT
+JKL
+JOE
+JPG
+JSF
+JSP
+JST
+JTA
+JVC
+JVM
+JYJ
+JYP
+Jac
+Jam
+Jan
+Jap
+Jav
+Jay
+Jin
+Joh
+Jon
+Jul
+Jun
+Jus
+KAB
+KAT
+KBS
+KDF
+KDJ
+KEY
+KFC
+KFR
+KID
+KIS
+KJm
+KOF
+KOH
+KOL
+KPI
+KPL
+KTV
+KVM
+Kin
+Kon
+Kur
+LAB
+LAN
+LBS
+LCA
+LCD
+LCK
+LCS
+LDA
+LDH
+LDL
+LDP
+LED
+LEE
+LEO
+LES
+LET
+LGA
+LGD
+LIN
+LIU
+LLC
+LME
+LMS
+LNG
+LOF
+LOL
+LOW
+LPG
+LPL
+LPR
+LRC
+LSA
+LSD
+LSI
+LSP
+LTD
+LTE
+LUC
+LUN
+LVM
+Laz
+Lib
+Lif
+Lin
+Liu
+Liz
+Lon
+Lou
+Low
+Luc
+Lum
+Luo
+Lux
+MAC
+MAD
+MAG
+MAN
+MAO
+MAP
+MAR
+MAS
+MAT
+MAX
+MAY
+MBA
+MBC
+MBO
+MBR
+MBS
+MCA
+MCC
+MCM
+MCN
+MCP
+MCS
+MCU
+MDA
+MDI
+MDL
+MDR
+MDS
+MEN
+MES
+MFA
+MFC
+MHC
+MHz
+MIB
+MIC
+MID
+MIL
+MIN
+MIS
+MIT
+MIX
+MKV
+MLC
+MLF
+MMA
+MMC
+MMI
+MMO
+MMS
+MMX
+MOD
+MOM
+MOS
+MOV
+MPA
+MPC
+MPG
+MPI
+MPS
+MPV
+MPa
+MRC
+MRI
+MRO
+MRP
+MSA
+MSC
+MSI
+MSN
+MTI
+MTK
+MTS
+MTU
+MTV
+MVC
+MVP
+Mac
+Mag
+Maj
+Man
+Mar
+Max
+May
+Mic
+Min
+Mon
+Mou
+Mur
+NAD
+NAS
+NAT
+NBA
+NBC
+NBL
+NCT
+NDS
+NEC
+NEO
+NES
+NET
+NEW
+NEX
+NFA
+NFC
+NFL
+NFS
+NGC
+NGN
+NGO
+NHK
+NHL
+NIC
+NIH
+NLP
+NME
+NMR
+NOT
+NOW
+NOX
+NOx
+NPC
+NPN
+NPR
+NSA
+NSC
+NSF
+NSK
+NTN
+NTP
+NTT
+NTV
+NVH
+NWA
+NXT
+NYT
+Nic
+Nob
+Nor
+Nov
+Now
+Nur
+OAD
+OBD
+OCG
+OCP
+OCR
+OCT
+ODM
+OEM
+OFF
+OGG
+OLE
+OMG
+ONE
+ONU
+OOO
+OPC
+OPP
+ORC
+OSD
+OSI
+OSS
+OST
+OTA
+OTC
+OTG
+OTT
+OUT
+OVA
+OVP
+Obj
+Off
+Oly
+Ope
+Oph
+Opt
+Our
+Out
+Ove
+PAC
+PAD
+PAH
+PAL
+PAM
+PAN
+PAS
+PBS
+PBT
+PCA
+PCB
+PCD
+PCI
+PCL
+PCM
+PCR
+PCS
+PCT
+PDA
+PDB
+PDC
+PDD
+PDF
+PDM
+PDP
+PDU
+PEG
+PEP
+PER
+PES
+PET
+PFA
+PFC
+PGA
+PGC
+PHP
+PHS
+PIC
+PID
+PIM
+PIN
+PKI
+PLA
+PLC
+PLD
+PLL
+PLM
+PMC
+PMI
+PMP
+PND
+PNG
+PNP
+POE
+POM
+PON
+POP
+POS
+PPA
+PPC
+PPG
+PPH
+PPI
+PPM
+PPP
+PPR
+PPS
+PPT
+PPV
+PRL
+PRO
+PSA
+PSD
+PSE
+PSG
+PSI
+PSK
+PSP
+PSS
+PSV
+PSW
+PSY
+PTA
+PTC
+PTH
+PTT
+PUB
+PVA
+PVC
+PVE
+PVP
+PWM
+Par
+Per
+Pic
+Pow
+Pro
+Pur
+QAM
+QDI
+QFP
+QGh
+QOS
+QPI
+QPS
+QRS
+QTL
+Qin
+Qua
+Que
+RAM
+RAP
+RAR
+RAS
+RAW
+RBC
+RCA
+RCS
+RDF
+RDS
+RED
+REF
+REG
+REM
+REX
+RFC
+RGB
+RIA
+RIM
+RIP
+RMB
+RMS
+RNA
+RNG
+ROC
+ROE
+ROI
+ROM
+RPC
+RPG
+RPM
+RRW
+RSA
+RSC
+RSI
+RSS
+RTA
+RTC
+RTK
+RTP
+RTS
+RTU
+RTX
+RUN
+RUS
+Ray
+Raz
+Ric
+Riv
+Rom
+Rou
+Rub
+Run
+Rus
+SAC
+SAE
+SAM
+SAN
+SAO
+SAP
+SAR
+SAS
+SAT
+SAY
+SBR
+SBS
+SCE
+SCH
+SCI
+SCM
+SCP
+SCR
+SDH
+SDI
+SDK
+SDR
+SDS
+SEA
+SEC
+SEE
+SEM
+SEO
+SER
+SET
+SFC
+SFP
+SGH
+SGI
+SGS
+SHA
+SHE
+SID
+SIG
+SIM
+SIP
+SIR
+SIS
+SKF
+SKT
+SKU
+SKY
+SLA
+SLC
+SLE
+SLG
+SLI
+SLR
+SLS
+SMA
+SMB
+SMC
+SMD
+SMG
+SMI
+SMP
+SMS
+SMT
+SNK
+SNP
+SNR
+SNS
+SOA
+SOC
+SOD
+SOI
+SOP
+SOS
+SPA
+SPC
+SPD
+SPE
+SPF
+SPI
+SPR
+SPS
+SPT
+SPV
+SQL
+SQU
+SRS
+SRT
+SSA
+SSC
+SSD
+SSE
+SSH
+SSL
+SSR
+SSS
+SST
+STC
+STD
+STK
+STL
+STM
+STN
+STP
+STR
+STS
+SUB
+SUN
+SUV
+SVC
+SVD
+SVG
+SVM
+SWF
+SXG
+SYN
+SYS
+Sch
+Ser
+She
+Siz
+Som
+Sou
+Squ
+Sub
+Sum
+Sun
+Sup
+Suz
+TAB
+TAC
+TAG
+TAO
+TBC
+TBM
+TBS
+TCG
+TCL
+TCM
+TCO
+TCP
+TCR
+TCS
+TCT
+TDD
+TDI
+TDM
+TDP
+TDS
+TEC
+TED
+TEL
+TEM
+TES
+TEU
+TEX
+TFT
+TGA
+TGF
+TGV
+THD
+THE
+TIA
+TIF
+TKO
+TLC
+TLS
+TMD
+TMP
+TMS
+TMT
+TNA
+TNF
+TNT
+TOC
+TOD
+TOE
+TOM
+TOP
+TPC
+TPE
+TPM
+TPO
+TPP
+TPR
+TPS
+TPU
+TQM
+TSC
+TSH
+TSI
+TSP
+TTL
+TTS
+TTT
+TUV
+TVB
+TVC
+TVP
+TVS
+TWO
+TXT
+Tay
+The
+Tom
+Tou
+Tow
+Tur
+UAR
+UBC
+UCC
+UCL
+UDP
+UFC
+UFO
+UGC
+UHF
+UIP
+UMD
+UML
+UNI
+UPC
+UPS
+URL
+USA
+USB
+USD
+USM
+USP
+USS
+UTC
+UTF
+UTP
+UTR
+UVA
+UVB
+UWB
+UZI
+Umb
+Uni
+Upp
+Uzi
+VAC
+VAR
+VBA
+VBR
+VBS
+VCC
+VCD
+VCR
+VDC
+VDE
+VGA
+VHF
+VHS
+VIA
+VII
+VIP
+VIS
+VMw
+VOA
+VOB
+VOC
+VOD
+VOL
+VPN
+VPS
+VRP
+VSS
+VTE
+VVT
+Ver
+Vic
+Vid
+Vis
+Viv
+WAN
+WAP
+WAV
+WAY
+WBA
+WBC
+WBO
+WBS
+WCG
+WCW
+WDM
+WDS
+WEB
+WEP
+WEY
+WGK
+WHO
+WIN
+WMA
+WMS
+WMV
+WOW
+WPA
+WPF
+WPS
+WRC
+WSA
+WTA
+WTI
+WTO
+WVG
+WWE
+WWF
+WWW
+Way
+Wha
+Whe
+Whi
+Who
+Why
+WiF
+Win
+Wiz
+Wom
+Wor
+Wou
+XGA
+XII
+XML
+XPS
+XXX
+XYZ
+YAG
+YES
+YOU
+YZB
+Yin
+You
+Yua
+Yuk
+Yun
+ZIP
+ZOL
+Zer
+Zha
+Zhu
+Zom
+Zon
+Zou
+abb
+abc
+abo
+abs
+act
+adj
+aff
+all
+and
+ang
+any
+app
+aws
+bbb
+bbc
+bbq
+bbs
+but
+cAM
+cDN
+cGM
+can
+car
+cba
+cha
+chi
+col
+com
+con
+cor
+cou
+cpi
+cpu
+dan
+day
+des
+did
+dif
+dis
+div
+diy
+doc
+don
+dow
+eAA
+eSA
+ech
+eff
+emb
+emp
+end
+eng
+eqc
+equ
+euv
+eve
+exc
+exe
+exp
+fac
+fil
+fin
+fir
+fiv
+fla
+fly
+for
+fox
+fre
+fri
+gAS
+gdp
+gen
+giv
+gmp
+gon
+goo
+got
+gps
+gra
+gre
+gro
+had
+har
+has
+hav
+haz
+her
+his
+hiv
+hol
+hom
+hou
+how
+iBT
+iOS
+iPa
+iPh
+iPo
+iSC
+ima
+imp
+inc
+inf
+inj
+int
+ipa
+iph
+ipo
+isb
+iso
+jam
+jap
+jav
+jay
+jus
+kHz
+kJm
+kdj
+kin
+lay
+laz
+lck
+lea
+led
+let
+lib
+lif
+lin
+liq
+lis
+lit
+liv
+liz
+lly
+lng
+loc
+lof
+log
+loo
+los
+low
+mRN
+mac
+mad
+maj
+man
+mar
+mat
+max
+may
+maz
+mba
+men
+mic
+min
+mmH
+mod
+mon
+mor
+mys
+nVI
+nba
+nex
+nic
+not
+nov
+now
+nxp
+obj
+off
+one
+ope
+opp
+our
+out
+ove
+par
+pay
+per
+phe
+php
+piz
+pla
+pow
+ppp
+pre
+pro
+pvc
+qHD
+qgh
+qua
+que
+qui
+rRN
+ray
+raz
+rea
+rec
+red
+ref
+reg
+rem
+rep
+req
+res
+rev
+ric
+riv
+rmb
+rng
+rom
+rou
+say
+sch
+sha
+she
+shi
+sho
+sim
+sin
+siz
+som
+sou
+spa
+spe
+sql
+squ
+sta
+ste
+sto
+str
+sty
+sub
+suv
+tRN
+tha
+the
+thi
+thr
+tim
+tip
+top
+tow
+tpp
+tra
+tur
+tuv
+two
+ubc
+uiv
+unc
+und
+uni
+unk
+ups
+usb
+uva
+uvb
+uzi
+val
+var
+ver
+vie
+vip
+vis
+viv
+wan
+was
+way
+web
+wer
+wha
+whi
+who
+why
+wif
+wit
+wom
+won
+wor
+wou
+www
+xin
+xxx
+yin
+you
+zha
+zhi
+zho
+zhu
+zon
+zzf
+zzy
+AAAA
+AACS
+ABCD
+ACCA
+ACCE
+ACCP
+ACDC
+ACGN
+ACID
+ACPI
+ACTH
+ADHD
+ADPC
+ADSL
+AIDS
+AJAX
+ALPH
+AMEX
+AMOL
+ANGE
+ANSI
+ANSY
+APEC
+APPL
+APTE
+ARDS
+ARPA
+ARPG
+ASCE
+ASCI
+ASIA
+ASIC
+ASIN
+ASME
+ASSO
+ASTM
+ASUS
+AUDI
+AUTO
+AVCH
+AWAR
+Andr
+BABY
+BACK
+BAND
+BANG
+BANK
+BASI
+BASS
+BATT
+BEAS
+BEAT
+BEST
+BEYO
+BIGB
+BIOS
+BLAC
+BLEA
+BLOG
+BLOO
+BLUE
+BOBO
+BOOK
+BOOL
+BOOM
+BOPP
+BOSS
+BOYS
+BRAV
+BREA
+BUFF
+Buck
+Buff
+Bull
+Bung
+Buzz
+CADC
+CALL
+CAPC
+CAPP
+CARD
+CASE
+CASI
+CAST
+CATI
+CATV
+CAXA
+CCFL
+CCIE
+CCNA
+CCTV
+CDMA
+CEPA
+CERN
+CHAN
+CHAP
+CHAR
+CHEN
+CHIN
+CHOR
+CIMS
+CIPA
+CISC
+CITE
+CITY
+CLAM
+CLAN
+CLAS
+CLOS
+CLUB
+CMMB
+CMMI
+CMOS
+CMYK
+CNAS
+CNBC
+CNBL
+CNKI
+CNNI
+COCO
+CODE
+COLL
+COLO
+COMB
+COME
+COMI
+COMP
+CONT
+COOL
+CORB
+CORE
+COSM
+COSP
+COST
+COUN
+COVI
+CPLD
+CREA
+CROS
+CSCD
+CSDN
+CSMA
+CSOL
+CSSC
+CSTN
+CTRL
+CUBA
+CUDA
+CURR
+CVBS
+Chin
+Chur
+DANC
+DARK
+DARP
+DASH
+DATA
+DAYS
+DCDC
+DDNS
+DDOS
+DDRI
+DELL
+DEMO
+DESI
+DEST
+DHCP
+DIGI
+DIMM
+DISC
+DIVX
+DLNA
+DOHC
+DOTA
+DOWN
+DRAG
+DRAM
+DREA
+DRIV
+DSLR
+DVDC
+DVGA
+DWDM
+DWOR
+EAST
+EASY
+EBIT
+ECMO
+EDGE
+EDIT
+EDTA
+EGFR
+EINE
+ELIS
+ELLE
+EMBA
+ENER
+ENGI
+ENTE
+EPDM
+EPIS
+EPON
+EPSO
+EPUB
+ERCP
+ERRO
+ESET
+ESPN
+ETSI
+EVDO
+EVER
+EXCE
+EXIL
+EXPO
+Ever
+Exch
+Exer
+FACE
+FALS
+FANS
+FANU
+FAST
+FDDI
+FIBA
+FIDI
+FIFA
+FIFO
+FILE
+FINA
+FIRE
+FIRS
+FISH
+FIVE
+FLAC
+FLAS
+FLOW
+FMVP
+FORT
+FPGA
+FREE
+FROM
+FTTH
+FULL
+FWVG
+FXCM
+Fuck
+Full
+Fund
+Fung
+Fuzz
+GABA
+GALA
+GAME
+GANK
+GATT
+GEAR
+GENE
+GHOS
+GIRL
+GLON
+GMAT
+GNSS
+GOLD
+GOOD
+GOOG
+GPRS
+GREE
+GROU
+GSMG
+GUCC
+GUND
+GUTS
+Gund
+HACC
+HAPP
+HARD
+HART
+HDCP
+HDMI
+HDPE
+HDTV
+HEAD
+HEAR
+HELL
+HEPA
+HERO
+HIFI
+HIGH
+HIPH
+HKEY
+HOLD
+HOME
+HOST
+HOUS
+HPLC
+HSDP
+HSPA
+HTML
+HTTP
+HUNT
+Hugh
+Hung
+ICAN
+ICMP
+ICON
+IDEA
+IDOL
+IEEE
+IELT
+IETF
+IFPI
+IGBT
+IGMP
+IMAX
+IMDB
+INFO
+INTE
+IPAD
+IPTV
+ISBN
+ISDN
+ISIS
+ISOI
+ISRC
+ISSN
+ISTP
+ITER
+ITIL
+IUCN
+Inte
+Inve
+JACK
+JAPA
+JAVA
+JAZZ
+JBOD
+JOHN
+JOJO
+JOKE
+JOUR
+JPEG
+JUMP
+JUST
+Jack
+Jake
+Jazz
+John
+Joke
+July
+Jump
+Jung
+KING
+KISS
+KONA
+KOYO
+LASI
+LAST
+LEED
+LEEP
+LESS
+LEVE
+LEXU
+LIFE
+LIKE
+LIMI
+LINE
+LINK
+LINU
+LIST
+LIVE
+LLDP
+LOCA
+LOFT
+LOGO
+LOLI
+LONG
+LOOK
+LOVE
+LPGA
+LTPS
+LVDS
+Ligh
+Like
+Lily
+Lind
+Ling
+Liqu
+Live
+Luck
+Luke
+MACD
+MACH
+MAGI
+MALL
+MAMA
+MARK
+MAST
+MATL
+MATX
+MAYA
+MBLA
+MEDI
+MEGA
+MEMS
+MERS
+META
+MIDI
+MIDP
+MIMO
+MINI
+MIPS
+MISS
+MIUI
+MMOR
+MOBA
+MODB
+MODE
+MOMO
+MOOC
+MOON
+MORE
+MOSF
+MOTO
+MOVI
+MPEG
+MPLS
+MSCI
+MSDS
+MTBF
+MUSI
+Mach
+Make
+Maur
+Mazz
+NACH
+NADH
+NADP
+NAMC
+NAME
+NANA
+NAND
+NASA
+NASD
+NATO
+NAVE
+NCAA
+NCAP
+NCIS
+NEDC
+NEOP
+NERV
+NEST
+NEWS
+NEXT
+NICO
+NIGH
+NIKE
+NINE
+NOKI
+NOTE
+NOVA
+NSAI
+NTFS
+NTSC
+NULL
+NURB
+NVID
+NYSE
+Nove
+ODBC
+OECD
+OFDM
+OFFI
+OLAP
+OLED
+ONLI
+ONLY
+OPEC
+OPEN
+OPPO
+ORAC
+ORIC
+ORIG
+OSPF
+OVER
+Oper
+PACS
+PAGE
+PARK
+PART
+PASS
+PCMC
+PDCA
+PEEK
+PERC
+PERF
+PETS
+PHEV
+PHIL
+PHOT
+PICC
+PIEC
+PLAN
+PLAY
+PLUS
+PMMA
+PNAS
+POLO
+POSE
+POST
+POWE
+PPTP
+PPTV
+PRAD
+PROD
+PROF
+PROJ
+PSTN
+PTFE
+PUNK
+PVDF
+Pric
+Prin
+Priv
+Priz
+Prom
+QFII
+QVGA
+QVOD
+QWER
+Quic
+Quin
+Quiz
+RADI
+RAID
+RAIN
+REAC
+READ
+REAL
+REIT
+RESE
+RFID
+RIDE
+RISC
+RMON
+RMRM
+RMVB
+ROAD
+ROCK
+ROHS
+ROOT
+ROSE
+RTEC
+RWBY
+Ruby
+SAAS
+SAMS
+SARS
+SATA
+SCAD
+SCAR
+SCDM
+SCHO
+SCIE
+SCSI
+SDHC
+SDMM
+SDRA
+SDSD
+SDXC
+SECA
+SECC
+SECT
+SEED
+SEGA
+SELE
+SERV
+SEVE
+SFDA
+SHIF
+SHIN
+SHOC
+SHOP
+SHOW
+SIDE
+SIEM
+SING
+SIZE
+SKIP
+SMAP
+SMAR
+SMIL
+SMTP
+SNMP
+SOAP
+SOCK
+SOHO
+SOLO
+SONG
+SONY
+SOSO
+SOUL
+SPAC
+SPCC
+SPDI
+SPEC
+SPEE
+SPIE
+SPOR
+SPSS
+SRAM
+SSCI
+STAF
+STAG
+STAR
+STAT
+STEM
+STEP
+STER
+STOP
+STOR
+STUD
+STYL
+SUMM
+SUPE
+SUSE
+SWAT
+SWIF
+SWOT
+SYST
+Subj
+Sull
+Sund
+Sung
+Supp
+TABL
+TANK
+TCPI
+TDMA
+TEAM
+TECH
+TEST
+TEXT
+TFBO
+TFSI
+TFTP
+THIS
+THRE
+TIFF
+TIME
+TIMK
+TIPS
+TOEF
+TOKY
+TOSH
+TOUC
+TOUR
+TOWN
+TRAC
+TRIP
+TRIZ
+TRUE
+TVBS
+TVOC
+TWIC
+TYPE
+Ther
+Thin
+Thom
+Thou
+UCLA
+UHMW
+ULTR
+UMTS
+UNES
+UNIT
+UNIV
+UNIX
+Unic
+Unit
+Univ
+VAIO
+VCCI
+VEGF
+VERS
+VHDL
+VIDE
+VIER
+VIII
+VISA
+VISI
+VIST
+VIVO
+VLAN
+VLSI
+VOCA
+VOGU
+VOIP
+VRay
+VSAT
+Vick
+Vill
+WANG
+WAPI
+WASD
+WAVE
+WCBA
+WCDM
+WEEK
+WEST
+WHAT
+WHIT
+WIFI
+WIND
+WITH
+WLAN
+WORD
+WORK
+WORL
+WQVG
+WXGA
+Wang
+Wher
+WiMA
+Will
+Wind
+Wing
+XBOX
+XBRL
+XHTM
+XVID
+XXXX
+YAMA
+YANG
+YEAH
+YONE
+YOUN
+YOUR
+YOYO
+Yong
+Your
+ZAFT
+ZARA
+ZERO
+ZGMF
+ZHAN
+ZONE
+Zhon
+Zhou
+abby
+abou
+andr
+appl
+baby
+back
+blic
+call
+char
+chic
+chin
+coff
+coll
+comb
+comm
+comp
+cond
+cons
+cont
+dick
+diff
+ding
+dock
+doin
+dong
+down
+ever
+exch
+find
+foll
+four
+from
+fron
+goin
+good
+goog
+gove
+hack
+hall
+hand
+hang
+happ
+have
+here
+high
+home
+into
+inve
+jack
+java
+jazz
+jump
+jung
+just
+know
+life
+ligh
+like
+lily
+ling
+liqu
+live
+lock
+logo
+lond
+long
+look
+love
+macd
+mach
+make
+mapp
+mmer
+nove
+okay
+only
+oper
+oppo
+othe
+over
+play
+pray
+pric
+prin
+priv
+priz
+prod
+prom
+quic
+real
+requ
+righ
+scho
+shou
+show
+some
+star
+stat
+stay
+stom
+subj
+such
+suff
+supp
+take
+than
+they
+thin
+thou
+toke
+uber
+unic
+univ
+upon
+usdj
+user
+usin
+vill
+vivo
+wake
+wall
+wang
+want
+wave
+were
+what
+when
+wifi
+will
+wind
+wing
+with
+work
+xing
+xxxx
+year
+your
+zhon
+China
+Inter
+Journ
+china
+every
+inter
+iphon
+thing
+think
+where
+which
+Univer
+univer
+Windows
+windows
+##A
+##B
+##C
+##D
+##E
+##F
+##G
+##H
+##I
+##J
+##K
+##L
+##M
+##N
+##O
+##P
+##Q
+##R
+##S
+##T
+##U
+##V
+##W
+##X
+##Y
+##Z
+##a
+##b
+##c
+##d
+##e
+##f
+##g
+##h
+##i
+##j
+##k
+##l
+##m
+##n
+##o
+##p
+##q
+##r
+##s
+##t
+##u
+##v
+##w
+##x
+##y
+##z
+##AA
+##AB
+##AC
+##AD
+##AE
+##AF
+##AG
+##AH
+##AI
+##AK
+##AL
+##AM
+##AN
+##AO
+##AP
+##AQ
+##AR
+##AS
+##AT
+##AV
+##AW
+##AX
+##AY
+##AZ
+##BA
+##BB
+##BC
+##BE
+##BG
+##BI
+##BM
+##BN
+##BO
+##BP
+##BR
+##BS
+##BT
+##BU
+##BY
+##CA
+##CB
+##CC
+##CD
+##CE
+##CF
+##CG
+##CH
+##CI
+##CK
+##CL
+##CM
+##CN
+##CO
+##CP
+##CR
+##CS
+##CT
+##CU
+##DA
+##DB
+##DC
+##DD
+##DE
+##DI
+##DL
+##DM
+##DN
+##DO
+##DP
+##DR
+##DS
+##DT
+##DU
+##DX
+##DY
+##EA
+##EB
+##EC
+##ED
+##EE
+##EF
+##EG
+##EI
+##EK
+##EL
+##EM
+##EN
+##EO
+##EP
+##ER
+##ES
+##ET
+##EV
+##EW
+##EX
+##EY
+##FA
+##FC
+##FD
+##FE
+##FF
+##FI
+##FL
+##FO
+##FP
+##FR
+##FS
+##FT
+##FU
+##FX
+##Fi
+##GA
+##GC
+##GE
+##GF
+##GH
+##GI
+##GL
+##GN
+##GO
+##GP
+##GR
+##GS
+##GU
+##GY
+##HA
+##HC
+##HD
+##HE
+##HG
+##HI
+##HM
+##HN
+##HO
+##HP
+##HR
+##HS
+##HT
+##IA
+##IB
+##IC
+##ID
+##IE
+##IF
+##IG
+##II
+##IK
+##IL
+##IM
+##IN
+##IO
+##IP
+##IR
+##IS
+##IT
+##IU
+##IV
+##IX
+##IZ
+##JI
+##JO
+##Jo
+##Ju
+##KA
+##KE
+##KI
+##KK
+##KO
+##KU
+##KY
+##LA
+##LC
+##LD
+##LE
+##LF
+##LG
+##LI
+##LK
+##LL
+##LM
+##LO
+##LP
+##LS
+##LT
+##LU
+##LV
+##LY
+##MA
+##MB
+##MC
+##MD
+##ME
+##MF
+##MI
+##ML
+##MM
+##MN
+##MO
+##MP
+##MR
+##MS
+##MT
+##MV
+##MY
+##NA
+##NC
+##ND
+##NE
+##NG
+##NI
+##NJ
+##NK
+##NN
+##NO
+##NP
+##NS
+##NT
+##NU
+##NX
+##NY
+##NZ
+##OB
+##OC
+##OD
+##OE
+##OF
+##OG
+##OH
+##OI
+##OK
+##OL
+##OM
+##ON
+##OO
+##OP
+##OR
+##OS
+##OT
+##OU
+##OV
+##OW
+##OX
+##PA
+##PC
+##PD
+##PE
+##PF
+##PG
+##PH
+##PI
+##PL
+##PM
+##PO
+##PP
+##PR
+##PS
+##PT
+##PU
+##QU
+##Qu
+##RA
+##RB
+##RC
+##RD
+##RE
+##RF
+##RG
+##RH
+##RI
+##RK
+##RL
+##RM
+##RN
+##RO
+##RP
+##RR
+##RS
+##RT
+##RU
+##RY
+##SA
+##SB
+##SC
+##SD
+##SE
+##SF
+##SH
+##SI
+##SK
+##SL
+##SM
+##SN
+##SO
+##SP
+##SS
+##ST
+##SU
+##SY
+##TA
+##TC
+##TD
+##TE
+##TH
+##TI
+##TM
+##TO
+##TP
+##TR
+##TS
+##TT
+##TU
+##TV
+##TY
+##Tu
+##UB
+##UC
+##UD
+##UE
+##UF
+##UG
+##UI
+##UK
+##UL
+##UM
+##UN
+##UP
+##UR
+##US
+##UT
+##VA
+##VB
+##VC
+##VD
+##VE
+##VI
+##VO
+##VP
+##VR
+##VT
+##WA
+##WC
+##WE
+##WF
+##WI
+##WL
+##WM
+##WO
+##WS
+##XA
+##XC
+##XE
+##XG
+##XO
+##XP
+##XT
+##XX
+##XY
+##YA
+##YE
+##YL
+##YO
+##YP
+##YS
+##YT
+##ZA
+##ZB
+##ZE
+##ZI
+##ZO
+##ZR
+##ZU
+##ZX
+##ZZ
+##ab
+##ag
+##al
+##am
+##an
+##ar
+##as
+##at
+##ax
+##ay
+##az
+##bi
+##bj
+##bl
+##bo
+##by
+##ce
+##ch
+##ci
+##ck
+##cq
+##ct
+##dj
+##ed
+##en
+##er
+##ew
+##ex
+##ff
+##fi
+##gh
+##gn
+##ha
+##he
+##ho
+##hz
+##ic
+##id
+##im
+##in
+##is
+##it
+##iv
+##ix
+##iz
+##jj
+##jo
+##ke
+##ky
+##kz
+##ld
+##le
+##lf
+##ll
+##ly
+##mb
+##mp
+##na
+##nc
+##nd
+##ng
+##nj
+##nk
+##nn
+##nt
+##nz
+##ob
+##oj
+##ok
+##ol
+##om
+##on
+##op
+##or
+##ou
+##ow
+##ox
+##ph
+##pp
+##pu
+##pv
+##qf
+##ql
+##qq
+##qu
+##re
+##rk
+##ro
+##ry
+##sh
+##sq
+##st
+##th
+##ty
+##ub
+##ul
+##um
+##un
+##ur
+##us
+##uv
+##ux
+##uz
+##ve
+##vi
+##wn
+##ws
+##ww
+##xp
+##xx
+##xy
+##zh
+##zy
+##zz
+##ACE
+##ACH
+##ACT
+##ADE
+##AGE
+##AIN
+##AME
+##AND
+##ANG
+##ANO
+##ANT
+##ARD
+##ARE
+##ASS
+##AST
+##ATE
+##BER
+##BLE
+##BOX
+##BSD
+##Bay
+##CAD
+##CAL
+##CAM
+##COM
+##CSE
+##DEO
+##DER
+##DIA
+##DNA
+##DSL
+##DVD
+##EAM
+##EAR
+##ECT
+##EEN
+##ENS
+##ENT
+##ERA
+##ERS
+##ESE
+##ESS
+##FTA
+##GER
+##GHT
+##GIS
+##IAL
+##IBA
+##IBU
+##ICE
+##ICS
+##IDE
+##INA
+##INE
+##ING
+##INT
+##INY
+##ION
+##IPS
+##ITE
+##IVE
+##KER
+##KON
+##LAY
+##LLA
+##LOR
+##MAN
+##MAS
+##MAX
+##MES
+##NAD
+##NAL
+##NCE
+##NET
+##NEY
+##NIC
+##NNA
+##OCK
+##ODE
+##OME
+##ONE
+##ORA
+##OWS
+##Off
+##PAC
+##PER
+##PRS
+##RAN
+##RIS
+##RNA
+##ROM
+##RON
+##ROR
+##SCO
+##SHI
+##SIC
+##SOL
+##SON
+##SQL
+##TAL
+##TED
+##TER
+##TML
+##TON
+##TRA
+##UND
+##UNG
+##UPA
+##USB
+##USE
+##VEL
+##VER
+##VGA
+##VID
+##WER
+##You
+##abl
+##aby
+##ach
+##ack
+##act
+##ain
+##ake
+##all
+##aly
+##anc
+##and
+##ang
+##ank
+##app
+##ard
+##ark
+##art
+##ary
+##ash
+##ath
+##auv
+##ave
+##avi
+##azi
+##azy
+##azz
+##bVI
+##bby
+##ber
+##bje
+##ble
+##cGI
+##cho
+##com
+##cqu
+##day
+##der
+##ebo
+##ect
+##ell
+##emb
+##enc
+##eng
+##ent
+##erJ
+##ern
+##erv
+##ery
+##eve
+##ews
+##exp
+##ext
+##ezy
+##fer
+##ffe
+##fic
+##for
+##gaz
+##ger
+##ght
+##gin
+##hen
+##her
+##hev
+##hin
+##hon
+##hou
+##iRF
+##ial
+##ica
+##ice
+##ich
+##ick
+##iff
+##igh
+##ike
+##ill
+##ily
+##ime
+##ine
+##ing
+##ink
+##ion
+##iqu
+##ish
+##ith
+##ive
+##iza
+##ize
+##izz
+##jin
+##ker
+##kin
+##lDR
+##lay
+##laz
+##lex
+##lic
+##lin
+##liz
+##llo
+##lly
+##man
+##maz
+##men
+##mer
+##min
+##mpl
+##mpo
+##nGL
+##nRH
+##nal
+##ner
+##ngz
+##niz
+##now
+##nxp
+##oCA
+##obj
+##ock
+##oll
+##omb
+##ome
+##omm
+##omp
+##one
+##ong
+##ook
+##ork
+##orm
+##ort
+##ory
+##oul
+##oup
+##our
+##ous
+##out
+##ove
+##own
+##ows
+##per
+##phe
+##ply
+##por
+##ppl
+##ppy
+##qqu
+##qua
+##que
+##qui
+##raz
+##rch
+##ric
+##rou
+##son
+##tBI
+##tch
+##ter
+##the
+##tic
+##tim
+##tiv
+##tur
+##uch
+##uck
+##uct
+##uff
+##ugh
+##umb
+##ung
+##ure
+##urn
+##vel
+##ven
+##ver
+##vic
+##vid
+##vin
+##war
+##way
+##whe
+##wor
+##www
+##xxx
+##ymb
+##yth
+##zhe
+##zym
+##zzy
+##ATIO
+##CESS
+##CIAT
+##CTIO
+##CTOR
+##ENGI
+##ERSI
+##HCSD
+##INES
+##INUE
+##IONA
+##LOID
+##MENT
+##NEER
+##NOLO
+##NTER
+##NTSC
+##ORMA
+##OSHO
+##RISE
+##RNAT
+##RNET
+##SATA
+##SION
+##TION
+##TTLE
+##VERS
+##ally
+##arch
+##ayer
+##azer
+##azin
+##bert
+##book
+##chin
+##ctor
+##ding
+##echn
+##erPC
+##erVR
+##eriz
+##erve
+##ever
+##ffer
+##ffff
+##ffic
+##fter
+##ghly
+##hell
+##ical
+##iche
+##icke
+##ific
+##ight
+##iver
+##izon
+##izzy
+##king
+##lack
+##land
+##llow
+##mber
+##ngin
+##ning
+##omic
+##onom
+##othe
+##ouch
+##ough
+##ound
+##ower
+##pper
+##ppin
+##pter
+##ster
+##ther
+##tion
+##tive
+##tter
+##ture
+##urch
+##vely
+##ction
+##ctive
+##enter
+##erica
+##ional
+##thing

bert/Erlangshen-MegatronBert-1.3B-Chinese/config.json

@@ -0,0 +1 @@
+{"vocab_size": 21248, "hidden_size": 2048, "num_hidden_layers": 24, "num_attention_heads": 8, "hidden_act": "gelu_new", "intermediate_size": 8192, "hidden_dropout_prob": 0.1, "attention_probs_dropout_prob": 0.1, "max_position_embeddings": 512, "type_vocab_size": 2, "initializer_range": 0.02, "layer_norm_eps": 1e-12, "gradient_checkpointing": false, "position_embedding_type": "absolute", "use_cache": false, "model_type": "megatron-bert"}

bert/Erlangshen-MegatronBert-3.9B-Chinese/config.json

@@ -0,0 +1,21 @@
+{
+    "architectures": [
+        "MegatronBertForMaskedLM"
+    ],
+    "vocab_size": 21248,
+    "hidden_size": 2560,
+    "num_hidden_layers": 48,
+    "num_attention_heads": 40,
+    "hidden_act": "gelu",
+    "intermediate_size": 10240,
+    "hidden_dropout_prob": 0.1,
+    "attention_probs_dropout_prob": 0.1,
+    "max_position_embeddings": 512,
+    "type_vocab_size": 2,
+    "initializer_range": 0.02,
+    "layer_norm_eps": 1e-12,
+    "gradient_checkpointing": false,
+    "position_embedding_type": "absolute",
+    "use_cache": false,
+    "model_type": "megatron-bert"
+}

bert/Erlangshen-MegatronBert-3.9B-Chinese/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/Erlangshen-MegatronBert-3.9B-Chinese/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "name_or_path": "/cognitive_comp/gaoxinyu/hf_hub/Erlangshen-MegatronBert-3.9B",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

bert/bert-base-japanese-v3/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/bert-base-japanese-v3/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert/bert-base-japanese-v3/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "BertForPreTraining"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 32768
+}

bert/bert-base-japanese-v3/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+    "tokenizer_class": "BertJapaneseTokenizer",
+    "model_max_length": 512,
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece",
+    "mecab_kwargs": {
+        "mecab_dic": "unidic_lite"
+    }
+}

bert/bert-large-japanese-v2/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/bert-large-japanese-v2/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT large Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT large model; 24 layers, 1024 dimensions of hidden states, and 16 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert/bert-large-japanese-v2/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "BertForPreTraining"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 32768
+}

bert/bert-large-japanese-v2/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+    "tokenizer_class": "BertJapaneseTokenizer",
+    "model_max_length": 512,
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece",
+    "mecab_kwargs": {
+        "mecab_dic": "unidic_lite"
+    }
+}

bert/bert_models.json

@@ -0,0 +1,14 @@
+{
+    "deberta-v2-large-japanese-char-wwm": {
+        "repo_id": "ku-nlp/deberta-v2-large-japanese-char-wwm",
+        "files": ["pytorch_model.bin"]
+    },
+    "chinese-roberta-wwm-ext-large": {
+        "repo_id": "hfl/chinese-roberta-wwm-ext-large",
+        "files": ["pytorch_model.bin"]
+    },
+    "deberta-v3-large": {
+        "repo_id": "microsoft/deberta-v3-large",
+        "files": ["spm.model", "pytorch_model.bin"]
+    }
+}

bert/chinese-roberta-wwm-ext-large/.gitattributes

@@ -0,0 +1,9 @@
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text

bert/chinese-roberta-wwm-ext-large/README.md

@@ -0,0 +1,57 @@
+---
+language:
+- zh
+tags:
+- bert
+license: "apache-2.0"
+---
+
+# Please use 'Bert' related functions to load this model!
+
+## Chinese BERT with Whole Word Masking
+For further accelerating Chinese natural language processing, we provide **Chinese pre-trained BERT with Whole Word Masking**.
+
+**[Pre-Training with Whole Word Masking for Chinese BERT](https://arxiv.org/abs/1906.08101)**
+Yiming Cui, Wanxiang Che, Ting Liu, Bing Qin, Ziqing Yang, Shijin Wang, Guoping Hu
+
+This repository is developed based on：https://github.com/google-research/bert
+
+You may also interested in,
+- Chinese BERT series: https://github.com/ymcui/Chinese-BERT-wwm
+- Chinese MacBERT: https://github.com/ymcui/MacBERT
+- Chinese ELECTRA: https://github.com/ymcui/Chinese-ELECTRA
+- Chinese XLNet: https://github.com/ymcui/Chinese-XLNet
+- Knowledge Distillation Toolkit - TextBrewer: https://github.com/airaria/TextBrewer
+
+More resources by HFL: https://github.com/ymcui/HFL-Anthology
+
+## Citation
+If you find the technical report or resource is useful, please cite the following technical report in your paper.
+- Primary: https://arxiv.org/abs/2004.13922
+```
+@inproceedings{cui-etal-2020-revisiting,
+    title = "Revisiting Pre-Trained Models for {C}hinese Natural Language Processing",
+    author = "Cui, Yiming  and
+      Che, Wanxiang  and
+      Liu, Ting  and
+      Qin, Bing  and
+      Wang, Shijin  and
+      Hu, Guoping",
+    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings",
+    month = nov,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.findings-emnlp.58",
+    pages = "657--668",
+}
+```
+- Secondary: https://arxiv.org/abs/1906.08101
+```
+@article{chinese-bert-wwm,
+  title={Pre-Training with Whole Word Masking for Chinese BERT},
+  author={Cui, Yiming and Che, Wanxiang and Liu, Ting and Qin, Bing and Yang, Ziqing and Wang, Shijin and Hu, Guoping},
+  journal={arXiv preprint arXiv:1906.08101},
+  year={2019}
+ }
+```

bert/chinese-roberta-wwm-ext-large/added_tokens.json

@@ -0,0 +1 @@
+{}

bert/chinese-roberta-wwm-ext-large/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "directionality": "bidi",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "type_vocab_size": 2,
+  "vocab_size": 21128
+}

bert/chinese-roberta-wwm-ext-large/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

bert/chinese-roberta-wwm-ext-large/tokenizer_config.json

@@ -0,0 +1 @@
+{"init_inputs": []}

bert/deberta-v2-large-japanese-char-wwm/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/deberta-v2-large-japanese-char-wwm/README.md

@@ -0,0 +1,89 @@
+---
+language: ja
+license: cc-by-sa-4.0
+library_name: transformers
+tags:
+  - deberta
+  - deberta-v2
+  - fill-mask
+  - character
+  - wwm
+datasets:
+  - wikipedia
+  - cc100
+  - oscar
+metrics:
+  - accuracy
+mask_token: "[MASK]"
+widget:
+    - text: "京都大学で自然言語処理を[MASK][MASK]する。"
+---
+
+# Model Card for Japanese character-level DeBERTa V2 large
+
+## Model description
+
+This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the Japanese portion of OSCAR.
+This model is trained with character-level tokenization and whole word masking.
+
+## How to use
+
+You can use this model for masked language modeling as follows:
+
+```python
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese-char-wwm')
+model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese-char-wwm')
+
+sentence = '京都大学で自然言語処理を[MASK][MASK]する。'
+encoding = tokenizer(sentence, return_tensors='pt')
+...
+```
+
+You can also fine-tune this model on downstream tasks.
+
+## Tokenization
+
+There is no need to tokenize texts in advance, and you can give raw texts to the tokenizer.
+The texts are tokenized into character-level tokens by [sentencepiece](https://github.com/google/sentencepiece).
+
+## Training data
+
+We used the following corpora for pre-training:
+
+- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
+- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
+- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
+
+Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
+Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
+
+## Training procedure
+
+We first segmented texts in the corpora into words using [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) for whole word masking.
+Then, we built a sentencepiece model with 22,012 tokens including all characters that appear in the training corpus.
+
+We tokenized raw corpora into character-level subwords using the sentencepiece model and trained the Japanese DeBERTa model using [transformers](https://github.com/huggingface/transformers) library.
+The training took 26 days using 16 NVIDIA A100-SXM4-40GB GPUs.
+
+The following hyperparameters were used during pre-training:
+
+- learning_rate: 1e-4
+- per_device_train_batch_size: 26
+- distributed_type: multi-GPU
+- num_devices: 16
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 3,328
+- max_seq_length: 512
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
+- lr_scheduler_type: linear schedule with warmup (lr = 0 at 300k steps)
+- training_steps: 260,000
+- warmup_steps: 10,000
+
+The accuracy of the trained model on the masked language modeling task was 0.795.
+The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
+
+## Acknowledgments
+
+This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of Large-Scale Japanese Language Models".
+For training models, we used the mdx: a platform for the data-driven future.

bert/deberta-v2-large-japanese-char-wwm/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 3,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.25.1",
+  "type_vocab_size": 0,
+  "vocab_size": 22012
+}

bert/deberta-v2-large-japanese-char-wwm/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/deberta-v2-large-japanese-char-wwm/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "do_subword_tokenize": true,
+  "do_word_tokenize": true,
+  "jumanpp_kwargs": null,
+  "mask_token": "[MASK]",
+  "mecab_kwargs": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "subword_tokenizer_type": "character",
+  "sudachi_kwargs": null,
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "unk_token": "[UNK]",
+  "word_tokenizer_type": "basic"
+}

bert/deberta-v2-large-japanese/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

bert/deberta-v2-large-japanese/README.md

@@ -0,0 +1,111 @@
+---
+language: ja
+license: cc-by-sa-4.0
+library_name: transformers
+tags:
+  - deberta
+  - deberta-v2
+  - fill-mask
+datasets:
+  - wikipedia
+  - cc100
+  - oscar
+metrics:
+  - accuracy
+mask_token: "[MASK]"
+widget:
+  - text: "京都 大学 で 自然 言語 処理 を [MASK] する 。"
+---
+
+# Model Card for Japanese DeBERTa V2 large
+
+## Model description
+
+This is a Japanese DeBERTa V2 large model pre-trained on Japanese Wikipedia, the Japanese portion of CC-100, and the
+Japanese portion of OSCAR.
+
+## How to use
+
+You can use this model for masked language modeling as follows:
+
+```python
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-large-japanese')
+model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-large-japanese')
+
+sentence = '京都 大学 で 自然 言語 処理 を [MASK] する 。'  # input should be segmented into words by Juman++ in advance
+encoding = tokenizer(sentence, return_tensors='pt')
+...
+```
+
+You can also fine-tune this model on downstream tasks.
+
+## Tokenization
+
+The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in
+advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each
+word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).
+
+## Training data
+
+We used the following corpora for pre-training:
+
+- Japanese Wikipedia (as of 20221020, 3.2GB, 27M sentences, 1.3M documents)
+- Japanese portion of CC-100 (85GB, 619M sentences, 66M documents)
+- Japanese portion of OSCAR (54GB, 326M sentences, 25M documents)
+
+Note that we filtered out documents annotated with "header", "footer", or "noisy" tags in OSCAR.
+Also note that Japanese Wikipedia was duplicated 10 times to make the total size of the corpus comparable to that of
+CC-100 and OSCAR. As a result, the total size of the training data is 171GB.
+
+## Training procedure
+
+We first segmented texts in the corpora into words using [Juman++](https://github.com/ku-nlp/jumanpp).
+Then, we built a sentencepiece model with 32000 tokens including words ([JumanDIC](https://github.com/ku-nlp/JumanDIC))
+and subwords induced by the unigram language model of [sentencepiece](https://github.com/google/sentencepiece).
+
+We tokenized the segmented corpora into subwords using the sentencepiece model and trained the Japanese DeBERTa model
+using [transformers](https://github.com/huggingface/transformers) library.
+The training took 36 days using 8 NVIDIA A100-SXM4-40GB GPUs.
+
+The following hyperparameters were used during pre-training:
+
+- learning_rate: 1e-4
+- per_device_train_batch_size: 18
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 2,304
+- max_seq_length: 512
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-06
+- lr_scheduler_type: linear schedule with warmup
+- training_steps: 300,000
+- warmup_steps: 10,000
+
+The accuracy of the trained model on the masked language modeling task was 0.799.
+The evaluation set consists of 5,000 randomly sampled documents from each of the training corpora.
+
+## Fine-tuning on NLU tasks
+
+We fine-tuned the following models and evaluated them on the dev set of JGLUE.
+We tuned learning rate and training epochs for each model and task
+following [the JGLUE paper](https://www.jstage.jst.go.jp/article/jnlp/30/1/30_63/_pdf/-char/ja).
+
+| Model                         | MARC-ja/acc | JSTS/pearson | JSTS/spearman | JNLI/acc | JSQuAD/EM | JSQuAD/F1 | JComQA/acc |
+|-------------------------------|-------------|--------------|---------------|----------|-----------|-----------|------------|
+| Waseda RoBERTa base           | 0.965       | 0.913        | 0.876         | 0.905    | 0.853     | 0.916     | 0.853      |
+| Waseda RoBERTa large (seq512) | 0.969       | 0.925        | 0.890         | 0.928    | 0.910     | 0.955     | 0.900      |
+| LUKE Japanese base*           | 0.965       | 0.916        | 0.877         | 0.912    | -         | -         | 0.842      |
+| LUKE Japanese large*          | 0.965       | 0.932        | 0.902         | 0.927    | -         | -         | 0.893      |
+| DeBERTaV2 base                | 0.970       | 0.922        | 0.886         | 0.922    | 0.899     | 0.951     | 0.873      |
+| DeBERTaV2 large               | 0.968       | 0.925        | 0.892         | 0.924    | 0.912     | 0.959     | 0.890      |
+
+*The scores of LUKE are from [the official repository](https://github.com/studio-ousia/luke).
+
+## Acknowledgments
+
+This work was supported by Joint Usage/Research Center for Interdisciplinary Large-scale Information Infrastructures (
+JHPCN) through General Collaboration Project no. jh221004, "Developing a Platform for Constructing and Sharing of
+Large-Scale Japanese Language Models".
+For training models, we used the mdx: a platform for the data-driven future.

bert/deberta-v2-large-japanese/config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "configs/deberta_v2_large.json",
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 3,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.23.1",
+  "type_vocab_size": 0,
+  "vocab_size": 32000
+}

bert/deberta-v2-large-japanese/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

bert/deberta-v2-large-japanese/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]"
+}