init

Dockerfile

@@ -0,0 +1,31 @@
+# create time : 2023.1.09
+# onnxruntime==1.13.1 not supported py=3.11
+# FROM python:3.11.1-slim-bullseye as compile-image
+FROM python:3.9.15-slim-bullseye as compile-image
+
+ENV POETRY_VERSION=1.3.1
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install cmake build-essential  -y --no-install-recommends && \
+    pip install poetry==$POETRY_VERSION 
+
+
+
+COPY ./pyproject.toml ./app/init_jptalk.py ./poetry.lock ./
+RUN poetry export -f requirements.txt -o requirements.txt --without dev --without test  --without-hashes  && \
+    python -m venv /opt/venv && \
+    /opt/venv/bin/pip install --no-cache-dir -U pip && \
+    /opt/venv/bin/pip install --no-cache-dir -r requirements.txt   && \
+    /opt/venv/bin/python3  init_jptalk.py 
+
+# FROM python:3.11.1-slim-bullseye as final
+FROM python:3.9.15-slim-bullseye as final
+EXPOSE 7860
+COPY --from=compile-image /opt/venv /opt/venv
+# COPY ./app/init_jptalk.py /app/init_jptalk.py
+ENV TZ=Asia/Shanghai PATH="/opt/venv/bin:$PATH"
+
+COPY ./app /app 
+WORKDIR /
+CMD ["python", "-m","app.main"]

LICENSE

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

README.md

@@ -1,11 +1,60 @@
----
-title: 19 Onnx
-emoji: 👀
-colorFrom: purple
-colorTo: blue
-sdk: docker
-pinned: false
-license: gpl-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## onnx inference server in docker container
+
+### Copy the demo web from [link](https://huggingface.co/spaces/skytnt/moe-japanese-tts/tree/main) 
+> Thanks a lot to [@CjangCjengh](https://github.com/CjangCjengh)
+> Thanks a lot to [wetts](https://github.com/wenet-e2e/wetts)
+
+***Only used for entertainment.
+Don't used for bussiness***
+
+### quick start 
+> To use other model and config<br> please use -v /path/to/dir:/app/.model to mount your model and config
+
+```shell
+export name=vits_onnx
+docker stop $name
+docker rm $name
+docker run -d \
+--name $name \
+-p 7860:7860 \
+ccdesue/vits_demo:onnx
+# -v /path/to/dir:/app/.model
+```
+
+
+
+
+### dir structure
+```
+
+├── app             # gradio code 
+├── build.sh
+├── Dockerfile      
+├── export          # some util for export model
+├── LICENSE
+├── poetry.lock
+├── __pycache__
+├── pyproject.toml
+├── README.md
+├── setup.sh
+└── util        # some posibile util 
+
+```
+
+### Helpful info
+1. please read the source code to better understand
+2. refer to the demo config.json to tail to your own model config
+3. refer the dockerfile 
+
+### limitation
+1. only  test  on japanese_cleaners and japanese_cleaners2 in config.json with  [raw vits](https://github.com/jaywalnut310/vits)
+
+
+### Reference
+1. [vits_export_discussion](https://github.com/MasayaKawamura/MB-iSTFT-VITS/issues/8)
+2. [other_vits_onnx](https://github.com/NaruseMioShirakana/VitsOnnx)
+3. [wetts](https://github.com/wenet-e2e/wetts)
+4. [android_vits](https://github.com/weirdseed/Vits-Android-ncnn)
+
+### license 
+GPLv2

app/__init__.py

@@ -0,0 +1,31 @@
+
+class HParams():
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if type(v) == dict:
+                v = HParams(**v)
+            self[k] = v
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __repr__(self):
+        return self.__dict__.__repr__()

app/config.py

@@ -0,0 +1,123 @@
+import os
+from pathlib import Path
+
+from loguru import logger
+# from app import CONFIG_URL, MODEL_URL
+from app.util import get_hparams_from_file, get_paths, time_it
+import requests
+from tqdm.auto import tqdm
+import re
+from re import Pattern
+import onnxruntime as ort
+import threading
+
+
+MODEL_URL = r"https://api.onedrive.com/v1.0/shares/u!aHR0cHM6Ly8xZHJ2Lm1zL3UvcyFBdG53cTVRejJnLTJmckZWcGdCR0xxLWJmU28/root/content"
+CONFIG_URL = r"https://api.onedrive.com/v1.0/shares/u!aHR0cHM6Ly8xZHJ2Lm1zL3UvcyFBdG53cTVRejJnLTJhNEJ3enhhUHpqNE5EZWc/root/content"
+
+
+
+class Config:
+    hps: dict = None
+    pattern: Pattern = None
+    # symbol_to_id:dict = None
+    speaker_choices: list = None
+    ort_sess: ort.InferenceSession = None
+    model_is_ok: bool = False
+
+    @classmethod
+    def init(cls):
+
+        # logger.add(
+        #     "vits_infer.log",  rotation="10 MB", encoding="utf-8", enqueue=True, retention="30 days"
+        # )
+
+        brackets = ['（', '[', '『', '「', '【', ")", "】", "]", "』", "」", "）"]
+        cls.pattern = re.compile('|'.join(map(re.escape, brackets)))
+
+        dir_path = Path(__file__).parent.absolute() / ".model"
+        dir_path.mkdir(
+            parents=True, exist_ok=True
+        )
+        model_path, config_path = get_paths(dir_path)
+
+        if not model_path or not config_path:
+            model_path = dir_path / "model.onnx"
+            config_path = dir_path / "config.json"
+            logger.warning(
+                "unable to find model or config, try to download default model and config"
+            )
+            cfg = requests.get(CONFIG_URL,  timeout=5).content
+            with open(str(config_path), 'wb') as f:
+                f.write(cfg)
+            cls.setup_config(str(config_path))
+            t = threading.Thread(target=cls.pdownload,
+                                 args=(MODEL_URL, str(model_path)))
+            t.start()
+            # cls.pdownload(MODEL_URL, str(model_path))
+
+        else:
+            cls.setup_config(str(config_path))
+            cls.setup_model(str(model_path))
+
+    @classmethod
+    @logger.catch
+    @time_it
+    def setup_model(cls, model_path: str):
+        import numpy as np
+        cls.ort_sess = ort.InferenceSession(model_path)
+        # init the model
+        seq = np.random.randint(low=0, high=len(
+            cls.hps.symbols), size=(1, 10), dtype=np.int64)
+
+        # seq_len = torch.IntTensor([seq.size(1)]).long()
+        seq_len = np.array([seq.shape[1]], dtype=np.int64)
+
+        # noise(可用于控制感情等变化程度) lenth(可用于控制整体语速) noisew(控制音素发音长度变化程度)
+        # 参考 https://github.com/gbxh/genshinTTS
+        # scales = torch.FloatTensor([0.667, 1.0, 0.8])
+        scales = np.array([0.667, 1.0, 0.8], dtype=np.float32)
+        # make triton dynamic shape happy
+        # scales = scales.unsqueeze(0)
+        scales.resize(1, 3)
+        # sid = torch.IntTensor([0]).long()
+        sid = np.array([0], dtype=np.int64)
+        # sid = torch.LongTensor([0])
+        ort_inputs = {
+            'input': seq,
+            'input_lengths': seq_len,
+            'scales': scales,
+            'sid': sid
+        }
+        cls.ort_sess.run(None, ort_inputs)
+
+        cls.model_is_ok = True
+
+        logger.info(
+            f"model init done with model path {model_path}"
+        )
+
+    @classmethod
+    def setup_config(cls, config_path: str):
+        cls.hps = get_hparams_from_file(config_path)
+        cls.speaker_choices = list(
+            map(lambda x: str(x[0])+":"+x[1], enumerate(cls.hps.speakers)))
+
+        logger.info(
+            f"config init done with config path {config_path}"
+        )
+
+    @classmethod
+    def pdownload(cls, url: str, save_path: str, chunk_size: int = 8192):
+        # copy from https://github.com/tqdm/tqdm/blob/master/examples/tqdm_requests.py
+        file_size = int(requests.head(url).headers["Content-Length"])
+        response = requests.get(url, stream=True)
+        with tqdm(total=file_size,  unit='B', unit_scale=True, unit_divisor=1024, miniters=1,
+                  desc="model download") as pbar:
+
+            with open(save_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        f.write(chunk)
+                        pbar.update(chunk_size)
+        cls.setup_model(save_path)

app/init_jptalk.py

@@ -0,0 +1,9 @@
+
+
+import pyopenjtalk
+import gradio as gr 
+
+
+
+pyopenjtalk. _lazy_init()
+# pyopenjtalk._extract_dic()

app/main.py

@@ -0,0 +1,94 @@
+# from multiprocessing import Process
+import numpy as np
+from .util import find_path_by_suffix, time_it
+from loguru import logger
+from .util import  intersperse
+from .config import Config
+from .text import text_to_sequence
+import gradio as gr
+# import sys
+# sys.path.append('..')
+
+
+def text_to_seq(text: str):
+    text = Config.pattern.sub(' ', text).strip()
+    text_norm = text_to_sequence(
+        text, Config.hps.symbols,  Config.hps.data.text_cleaners)
+    if Config.hps.data.add_blank:
+        text_norm = intersperse(text_norm, 0)
+    return text_norm
+
+
+@time_it
+@logger.catch
+def tts_fn(text, speaker_id, speed=1.0):
+
+    if len(text) > 300:
+        return "Error: Text is too long, please down it to 300 characters", None
+
+    if not Config.model_is_ok:
+        return "Error: model not loaded, please wait for a while or look the log", None
+
+    seq = text_to_seq(text)
+    x = np.array([seq], dtype=np.int64)
+    x_len = np.array([x.shape[1]], dtype=np.int64)
+    sid = np.array([speaker_id], dtype=np.int64)
+    speed = 1/speed
+    scales = np.array([0.667, speed, 0.8], dtype=np.float32)
+    scales.resize(1, 3)
+    ort_inputs = {
+        'input': x,
+        'input_lengths': x_len,
+        'scales': scales,
+        'sid': sid
+    }
+    audio = np.squeeze(Config.ort_sess.run(None, ort_inputs))
+    audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+    audio = np.clip(audio, -32767.0, 32767.0)
+
+    return "success", (Config.hps.data.sampling_rate, audio.astype(np.int16))
+
+
+def set_gradio_view():
+    app = gr.Blocks()
+
+    with app:
+        gr.Markdown(
+            "a demo of web service of vits, thanks to @CjangCjengh, copy from [link](https://huggingface.co/spaces/skytnt/moe-japanese-tts)")
+        with gr.Tabs():
+            with gr.TabItem("TTS"):
+                with gr.Column():
+                    tts_input1 = gr.TextArea(
+                        label="TTS_text", value="わたしの趣味はたくさんあります。でも、一番好きな事は写真をとることです。")
+                    tts_input2 = gr.Dropdown(
+                        label="Speaker", choices=Config.speaker_choices, type="index", value=Config.speaker_choices[0])
+                    tts_input3 = gr.Slider(
+                        label="Speed", value=1, minimum=0.2, maximum=3, step=0.1)
+
+                    tts_submit = gr.Button("Generate", variant="primary")
+                    tts_output1 = gr.Textbox(label="Output Message")
+                    tts_output2 = gr.Audio(label="Output Audio")
+
+                    inputs = [
+                        tts_input1, tts_input2, tts_input3
+                    ]
+                    outputs = [
+                        tts_output1, tts_output2]
+
+        tts_submit.click(tts_fn, inputs=inputs, outputs=outputs)
+
+    app.queue(concurrency_count=3)
+    gr.close_all()
+    app.launch(server_name='0.0.0.0', show_api=False,
+               share=False, server_port=7860)
+
+
+def main():
+    # p = Process(target=Config.init)
+    # p.start()
+    Config.init()
+    set_gradio_view()
+
+
+if __name__ == '__main__':
+    main()

app/text/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Keith Ito
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

app/text/__init__.py

@@ -0,0 +1,49 @@
+""" from https://github.com/keithito/tacotron """
+from loguru import logger
+# from app.config import Config
+from . import cleaners
+
+_symbol_to_id = None
+
+def text_to_sequence(text, symbols, cleaner_names):
+    '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+      Args:
+        text: string to convert to a sequence
+        symbols: list of symbols in the text
+        cleaner_names: names of the cleaner functions to run the text through
+      Returns:
+        List of integers corresponding to the symbols in the text
+
+
+        ATTENTION: unable to access Config variabel , don't know why
+    '''
+
+    global _symbol_to_id
+
+
+    if not _symbol_to_id:
+        _symbol_to_id = {s: i for i, s in enumerate(symbols)}
+
+ 
+
+    clean_text = _clean_text(text, cleaner_names)
+
+    sequence = [
+        _symbol_to_id[symbol] for symbol in clean_text if symbol in _symbol_to_id.keys()
+    ]
+
+    # for symbol in clean_text:
+    #     if symbol not in _symbol_to_id.keys():
+    #         continue
+    #     symbol_id = _symbol_to_id[symbol]
+    #     sequence += [symbol_id]
+    return sequence
+
+
+def _clean_text(text, cleaner_names):
+    for name in cleaner_names:
+        cleaner = getattr(cleaners, name)
+        if not cleaner:
+            raise Exception('Unknown cleaner: %s' % name)
+        text = cleaner(text)
+    return text

app/text/cleaners.py

@@ -0,0 +1,58 @@
+import re
+from unidecode import unidecode
+import pyopenjtalk
+
+pyopenjtalk._lazy_init()
+
+# Regular expression matching Japanese without punctuation marks:
+_japanese_characters = re.compile(
+    r'[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
+
+# Regular expression matching non-Japanese characters or punctuation marks:
+_japanese_marks = re.compile(
+    r'[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
+
+
+def japanese_cleaners(text):
+    '''Pipeline for notating accent in Japanese text.'''
+    '''Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html'''
+    sentences = re.split(_japanese_marks, text)
+    marks = re.findall(_japanese_marks, text)
+    text = ''
+    for i, sentence in enumerate(sentences):
+        if re.match(_japanese_characters, sentence):
+            if text != '':
+                text += ' '
+            labels = pyopenjtalk.extract_fullcontext(sentence)
+            for n, label in enumerate(labels):
+                phoneme = re.search(r'\-([^\+]*)\+', label).group(1)
+                if phoneme not in ['sil', 'pau']:
+                    text += phoneme.replace('ch', 'ʧ').replace('sh', 'ʃ').replace('cl', 'Q')
+                else:
+                    continue
+                n_moras = int(re.search(r'/F:(\d+)_', label).group(1))
+                a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
+                a2 = int(re.search(r"\+(\d+)\+", label).group(1))
+                a3 = int(re.search(r"\+(\d+)/", label).group(1))
+                if re.search(r'\-([^\+]*)\+', labels[n + 1]).group(1) in ['sil', 'pau']:
+                    a2_next = -1
+                else:
+                    a2_next = int(re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
+                # Accent phrase boundary
+                if a3 == 1 and a2_next == 1:
+                    text += ' '
+                # Falling
+                elif a1 == 0 and a2_next == a2 + 1 and a2 != n_moras:
+                    text += '↓'
+                # Rising
+                elif a2 == 1 and a2_next == 2:
+                    text += '↑'
+        if i < len(marks):
+            text += unidecode(marks[i]).replace(' ', '')
+    if re.match('[A-Za-z]', text[-1]):
+        text += '.'
+    return text
+
+
+def japanese_cleaners2(text):
+    return japanese_cleaners(text).replace('ts','ʦ').replace('...','…')

app/text/symbols.py

@@ -0,0 +1,39 @@
+'''
+Defines the set of symbols used in text input to the model.
+'''
+
+'''# japanese_cleaners
+_pad        = '_'
+_punctuation = ',.!?-'
+_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
+'''
+# jp_cleaners
+_pad        = '_'
+_punctuation = ',.!?-'
+_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
+
+
+
+# japanese_cleaners2
+# _pad        = '_'
+# _punctuation = ',.!?-~…'
+# _letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
+
+
+'''# korean_cleaners
+_pad        = '_'
+_punctuation = ',.!?…~'
+_letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
+'''
+
+'''# chinese_cleaners
+_pad        = '_'
+_punctuation = '，。！？—…'
+_letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
+'''
+
+# Export all symbols:
+symbols = [_pad] + list(_punctuation) + list(_letters)
+
+# Special symbol ids
+SPACE_ID = symbols.index(" ")

app/util.py

@@ -0,0 +1,86 @@
+
+import json
+import pathlib
+# import tqdm
+
+from typing import Optional
+import os
+import threading
+
+from loguru import logger
+# from app.common import HParams
+# from __ini import HParams
+from pathlib import Path
+import requests
+
+from app import HParams
+
+
+def find_path_by_suffix(dir_path: Path, suffix: Path):
+    assert dir_path.is_dir()
+
+    for path in dir_path.glob(f"*.{suffix}"):
+        return path
+
+    return None
+
+
+def get_hparams_from_file(config_path):
+    with open(config_path, "r") as f:
+        data = f.read()
+    config = json.loads(data)
+
+    hparams = HParams(**config)
+    return hparams
+
+
+def intersperse(lst, item):
+    result = [item] * (len(lst) * 2 + 1)
+    result[1::2] = lst
+    return result
+
+
+def time_it(func: callable):
+    import time
+
+    def wrapper(*args, **kwargs):
+        # start = time.time()
+        start = time.perf_counter()
+        res = func(*args, **kwargs)
+        # end = time.time()
+        end = time.perf_counter()
+        # print(f"func {func.__name__} cost {end-start} seconds")
+        logger.info(f"func {func.__name__} cost {end-start} seconds")
+        return res
+    return wrapper
+
+
+
+
+
+# def download_defaults(model_path: pathlib.Path, config_path: pathlib.Path):
+
+#     config = requests.get(config_url,  timeout=10).content
+#     with open(str(config_path), 'wb') as f:
+#         f.write(config)
+
+#     t = threading.Thread(target=pdownload, args=(model_url, str(model_path)))
+#     t.start()
+
+
+def get_paths(dir_path: Path):
+
+    model_path: Path = find_path_by_suffix(dir_path, "onnx")
+    config_path: Path = find_path_by_suffix(dir_path, "json")
+    # if not model_path or not config_path:
+    #     model_path = dir_path / "model.onnx"
+    #     config_path = dir_path / "config.json"
+    #     logger.warning(
+    #         "unable to find model or config, try to download default model and config"
+    #     )
+    #     download_defaults(model_path, config_path)
+
+    # model_path = str(model_path)
+    # config_path = str(config_path)
+    # logger.info(f"model path: {model_path} config path: {config_path}")
+    return model_path, config_path

build.sh

@@ -0,0 +1,20 @@
+docker image rm ccdesue/vits_demo:onnx -f --no-prune
+docker build -t ccdesue/vits_demo:onnx .
+
+function run()
+{
+export name=vits_onnx
+docker stop $name
+docker rm $name
+docker run -d \
+--name $name \
+-p 7860:7860 \
+ccdesue/vits_demo:onnx
+}
+
+# docker run --rm -it -p 7860:7860/tcp ccdesue/vits_demo:onnx  bash
+
+function push(){
+    
+    docker push ccdesue/vits_demo:onnx
+}

export/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

export/README.md

@@ -0,0 +1,8 @@
+> Thanks a lot to [wetts](https://github.com/wenet-e2e/wetts)
+> 欢迎 pr
+## 修改说明
+1. 将原仓库的配置文件修改成[@CjangCjengh](https://github.com/CjangCjengh)用的部署文件 详细参考config.json
+2. 为导出代码添加注释， tensor修改为np.array
+3. 有问题请认真阅读源码
+
+

export/export.sh

@@ -0,0 +1,19 @@
+
+function get_data(){
+    mkdir -p model 
+    cd model 
+    model_url='https://api.onedrive.com/v1.0/shares/u!aHR0cHM6Ly8xZHJ2Lm1zL3UvcyFBdG53cTVRejJnLTJiTzdqanlEQXNyWDV4bDA/root/content'
+    config_url='https://api.onedrive.com/v1.0/shares/u!aHR0cHM6Ly8xZHJ2Lm1zL3UvcyFBdG53cTVRejJnLTJhNEJ3enhhUHpqNE5EZWc/root/content'
+    
+    wget -O model.pth $model_url
+    wget -O  config.json $config_url
+    cd ..
+}
+
+
+# mkdir -p model
+
+python vits/export_onnx.py --checkpoint model/model.pth --cfg model/config.json \
+    --onnx_model model/model.onnx 
+
+# https://api.onedrive.com/v1.0/shares/u!aHR0cHM6Ly8xZHJ2Lm1zL3UvcyFBb1E3R21YN3hRWkpnYjQtT1VocVdjUFc4VWM5bVE/root/content

export/infer.sh

@@ -0,0 +1,2 @@
+python vits/inference_onnx.py --onnx_model model/model.onnx \
+--cfg model/config.json  --test_file test.txt

export/requirements.txt

@@ -0,0 +1,21 @@
+tqdm
+flake8==3.8.2
+flake8-bugbear
+flake8-comprehensions
+flake8-executable
+flake8-pyi==20.5.0
+# mccabe
+pycodestyle==2.6.0
+pyflakes==2.2.0
+# tensorboard
+sklearn
+WeTextProcessing
+monotonic_align
+matplotlib
+librosa
+scipy
+transformers
+# Cython
+pyopenjtalk
+unidecode
+# pip3 install torch --extra-index-url https://download.pytorch.org/whl/cpu

export/setup.sh

@@ -0,0 +1,18 @@
+# update pip 
+
+function setup_py(){
+conda create -n dl python=3.9 -y
+conda init bash
+bash
+}
+
+conda activate dl 
+export DEBIAN_FRONTEND=noninteractive && \
+    sudo apt-get update && \
+    sudo apt-get install cmake build-essential  -y --no-install-recommends
+
+
+pip install --upgrade pip
+pip3 install torch --extra-index-url https://download.pytorch.org/whl/cpu
+pip3 install onnxruntime Cython
+pip install -r requirements.txt

export/test.txt

@@ -0,0 +1 @@
+test.wav|わたしの趣味はたくさんあります。でも、一番好きな事は写真をとることです。

export/vits/attentions.py

@@ -0,0 +1,392 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import commons
+from modules import LayerNorm
+
+
+class Encoder(nn.Module):
+    def __init__(self,
+                 hidden_channels,
+                 filter_channels,
+                 n_heads,
+                 n_layers,
+                 kernel_size=1,
+                 p_dropout=0.,
+                 window_size=4,
+                 **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout,
+                                   window_size=window_size))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class Decoder(nn.Module):
+    def __init__(self,
+                 hidden_channels,
+                 filter_channels,
+                 n_heads,
+                 n_layers,
+                 kernel_size=1,
+                 p_dropout=0.,
+                 proximal_bias=False,
+                 proximal_init=True,
+                 **kwargs):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout,
+                                   proximal_bias=proximal_bias,
+                                   proximal_init=proximal_init))
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(hidden_channels,
+                                   hidden_channels,
+                                   n_heads,
+                                   p_dropout=p_dropout))
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                    causal=True))
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
+    x: decoder input
+    h: encoder output
+    """
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
+            device=x.device, dtype=x.dtype)
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self,
+                 channels,
+                 out_channels,
+                 n_heads,
+                 p_dropout=0.,
+                 window_size=None,
+                 heads_share=True,
+                 block_length=None,
+                 proximal_bias=False,
+                 proximal_init=False):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev)
+            self.emb_rel_v = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev)
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels,
+                           t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels,
+                           t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels),
+                              key.transpose(-2, -1))
+        if self.window_size is not None:
+            msg = "Relative attention is only available for self-attention."
+            assert t_s == t_t, msg
+            key_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(
+                query / math.sqrt(self.k_channels), key_relative_embeddings)
+            scores_local = self._relative_position_to_absolute_position(
+                rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            msg = "Proximal bias is only available for self-attention."
+            assert t_s == t_t, msg
+            scores = scores + self._attention_bias_proximal(t_s).to(
+                device=scores.device, dtype=scores.dtype)
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                msg = "Local attention is only available for self-attention."
+                assert t_s == t_t, msg
+                block_mask = torch.ones_like(scores).triu(
+                    -self.block_length).tril(self.block_length)
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(
+                p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_v, t_s)
+            output = output + self._matmul_with_relative_values(
+                relative_weights, value_relative_embeddings)
+        output = output.transpose(2, 3).contiguous().view(
+            b, d, t_t)  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
+    x: [b, h, l, m]
+    y: [h or 1, m, d]
+    ret: [b, h, l, d]
+    """
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
+
+    def _matmul_with_relative_keys(self, x, y):
+        """
+    x: [b, h, l, d]
+    y: [h or 1, m, d]
+    ret: [b, h, l, m]
+    """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        max_relative_position = 2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length],
+                                           [0, 0]]))
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[:,
+                                                              slice_start_position:
+                                                              slice_end_position]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
+    x: [b, h, l, 2*l-1]
+    ret: [b, h, l, l]
+    """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0,
+                                                                         1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0,
+                                                                length - 1]]))
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1,
+                               2 * length - 1])[:, :, :length, length - 1:]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
+    x: [b, h, l, l]
+    ret: [b, h, l, 2*l-1]
+    """
+        batch, heads, length, _ = x.size()
+        # padd along column
+        x = F.pad(
+            x,
+            commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0,
+                                                                length - 1]]))
+        x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
+    Args:
+      length: an integer scalar.
+    Returns:
+      a Tensor with shape [1, 1, length, length]
+    """
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(
+            torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout=0.,
+                 activation=None,
+                 causal=False):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

export/vits/commons.py

@@ -0,0 +1,161 @@
+import math
+
+import torch
+from torch.nn import functional as F
+
+
+def init_weights(m, mean=0.0, std=0.01):
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size * dilation - dilation) / 2)
+
+
+def convert_pad_shape(pad_shape):
+    pad_shape = [item for sublist in reversed(pad_shape) for item in sublist]
+    return pad_shape
+
+
+def intersperse(lst, item):
+    result = [item] * (len(lst) * 2 + 1)
+    result[1::2] = lst
+    return result
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+    """KL(P||Q)"""
+    kl = (logs_q - logs_p) - 0.5
+    kl += 0.5 * (torch.exp(2. * logs_p) +
+                 ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
+    return kl
+
+
+def rand_gumbel(shape):
+    """Sample from the Gumbel distribution, protect from overflows."""
+    uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+    return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+    g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+    return g
+
+
+def slice_segments(x, ids_str, segment_size=4):
+    ret = torch.zeros_like(x[:, :, :segment_size])
+    for i in range(x.size(0)):
+        idx_str = ids_str[i]
+        idx_end = idx_str + segment_size
+        ret[i] = x[i, :, idx_str:idx_end]
+    return ret
+
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+    b, d, t = x.size()
+    if x_lengths is None:
+        x_lengths = t
+    ids_str_max = x_lengths - segment_size + 1
+    ids_str = (torch.rand([b]).to(device=x.device) *
+               ids_str_max).to(dtype=torch.long)
+    ret = slice_segments(x, ids_str, segment_size)
+    return ret, ids_str
+
+
+def get_timing_signal_1d(length,
+                         channels,
+                         min_timescale=1.0,
+                         max_timescale=1.0e4):
+    position = torch.arange(length, dtype=torch.float)
+    num_timescales = channels // 2
+    log_timescale_increment = (
+        math.log(float(max_timescale) / float(min_timescale)) /
+        (num_timescales - 1))
+    inv_timescales = min_timescale * torch.exp(
+        torch.arange(num_timescales, dtype=torch.float) *
+        -log_timescale_increment)
+    scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+    signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+    signal = F.pad(signal, [0, 0, 0, channels % 2])
+    signal = signal.view(1, channels, length)
+    return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale,
+                                  max_timescale)
+    return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale,
+                                  max_timescale)
+    return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
+
+
+def subsequent_mask(length):
+    mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+    return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
+
+
+def shift_1d(x):
+    x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+    return x
+
+
+def sequence_mask(length, max_length=None):
+    if max_length is None:
+        max_length = length.max()
+    x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+    return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration, mask):
+    """
+  duration: [b, 1, t_x]
+  mask: [b, 1, t_y, t_x]
+  """
+    device = duration.device
+
+    b, _, t_y, t_x = mask.shape
+    cum_duration = torch.cumsum(duration, -1)
+
+    cum_duration_flat = cum_duration.view(b * t_x)
+    path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+    path = path.view(b, t_x, t_y)
+    path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]
+                                                 ]))[:, :-1]
+    path = path.unsqueeze(1).transpose(2, 3) * mask
+    return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+    if isinstance(parameters, torch.Tensor):
+        parameters = [parameters]
+    parameters = list(filter(lambda p: p.grad is not None, parameters))
+    norm_type = float(norm_type)
+    if clip_value is not None:
+        clip_value = float(clip_value)
+
+    total_norm = 0
+    for p in parameters:
+        param_norm = p.grad.data.norm(norm_type)
+        total_norm += param_norm.item()**norm_type
+        if clip_value is not None:
+            p.grad.data.clamp_(min=-clip_value, max=clip_value)
+    total_norm = total_norm**(1. / norm_type)
+    return total_norm

export/vits/data_utils.py

@@ -0,0 +1,307 @@
+import os
+import random
+
+import torch
+import torchaudio
+import torch.utils.data
+
+import commons
+from mel_processing import spectrogram_torch
+from utils import load_filepaths_and_text
+
+
+class TextAudioSpeakerLoader(torch.utils.data.Dataset):
+    """
+        1) loads audio, speaker_id, text pairs
+        2) normalizes text and converts them to sequences of integers
+        3) computes spectrograms from audio files.
+    """
+    def __init__(self, audiopaths_sid_text, hparams):
+        self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
+        # self.text_cleaners = hparams.text_cleaners
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length = hparams.filter_length
+        self.hop_length = hparams.hop_length
+        self.win_length = hparams.win_length
+        self.sampling_rate = hparams.sampling_rate
+        self.src_sampling_rate = getattr(hparams, "src_sampling_rate",
+                                         self.sampling_rate)
+
+        self.cleaned_text = getattr(hparams, "cleaned_text", False)
+
+        self.add_blank = hparams.add_blank
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 190)
+
+        phone_file = getattr(hparams, "phone_table", None)
+        self.phone_dict = None
+        if phone_file is not None:
+            self.phone_dict = {}
+            with open(phone_file) as fin:
+                for line in fin:
+                    arr = line.strip().split()
+                    self.phone_dict[arr[0]] = int(arr[1])
+
+        speaker_file = getattr(hparams, "speaker_table", None)
+        self.speaker_dict = None
+        if speaker_file is not None:
+            self.speaker_dict = {}
+            with open(speaker_file) as fin:
+                for line in fin:
+                    arr = line.strip().split()
+                    self.speaker_dict[arr[0]] = int(arr[1])
+
+        random.seed(1234)
+        random.shuffle(self.audiopaths_sid_text)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+
+        audiopaths_sid_text_new = []
+        lengths = []
+        for item in self.audiopaths_sid_text:
+            audiopath = item[0]
+            # filename|text or filename|speaker|text
+            text = item[1] if len(item) == 2 else item[2]
+            if self.min_text_len <= len(text) and len(
+                    text) <= self.max_text_len:
+                audiopaths_sid_text_new.append(item)
+                lengths.append(
+                    int(
+                        os.path.getsize(audiopath) * self.sampling_rate /
+                        self.src_sampling_rate) // (2 * self.hop_length))
+        self.audiopaths_sid_text = audiopaths_sid_text_new
+        self.lengths = lengths
+
+    def get_audio_text_speaker_pair(self, audiopath_sid_text):
+        audiopath = audiopath_sid_text[0]
+        if len(audiopath_sid_text) == 2:  # filename|text
+            sid = 0
+            text = audiopath_sid_text[1]
+        else:  # filename|speaker|text
+            sid = self.speaker_dict[audiopath_sid_text[1]]
+            text = audiopath_sid_text[2]
+        text = self.get_text(text)
+        spec, wav = self.get_audio(audiopath)
+        sid = self.get_sid(sid)
+        return (text, spec, wav, sid)
+
+    def get_audio(self, filename):
+        audio, sampling_rate = torchaudio.load(filename, normalize=False)
+        if sampling_rate != self.sampling_rate:
+            audio = audio.to(torch.float)
+            audio = torchaudio.transforms.Resample(sampling_rate,
+                                                   self.sampling_rate)(audio)
+            audio = audio.to(torch.int16)
+        audio = audio[0]  # Get the first channel
+        audio_norm = audio / self.max_wav_value
+        audio_norm = audio_norm.unsqueeze(0)
+        spec = spectrogram_torch(audio_norm,
+                                 self.filter_length,
+                                 self.sampling_rate,
+                                 self.hop_length,
+                                 self.win_length,
+                                 center=False)
+        spec = torch.squeeze(spec, 0)
+        return spec, audio_norm
+
+    def get_text(self, text):
+        text_norm = [self.phone_dict[phone] for phone in text.split()]
+        if self.add_blank:
+            text_norm = commons.intersperse(text_norm, 0)
+        text_norm = torch.LongTensor(text_norm)
+        return text_norm
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def __getitem__(self, index):
+        return self.get_audio_text_speaker_pair(
+            self.audiopaths_sid_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_sid_text)
+
+
+class TextAudioSpeakerCollate():
+    """ Zero-pads model inputs and targets
+    """
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text, audio and speaker identities
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized, sid]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(torch.LongTensor(
+            [x[1].size(1) for x in batch]),
+            dim=0,
+            descending=True)
+
+        max_text_len = max([len(x[0]) for x in batch])
+        max_spec_len = max([x[1].size(1) for x in batch])
+        max_wav_len = max([x[2].size(1) for x in batch])
+
+        text_lengths = torch.LongTensor(len(batch))
+        spec_lengths = torch.LongTensor(len(batch))
+        wav_lengths = torch.LongTensor(len(batch))
+        sid = torch.LongTensor(len(batch))
+
+        text_padded = torch.LongTensor(len(batch), max_text_len)
+        spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0),
+                                        max_spec_len)
+        wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
+        text_padded.zero_()
+        spec_padded.zero_()
+        wav_padded.zero_()
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            text = row[0]
+            text_padded[i, :text.size(0)] = text
+            text_lengths[i] = text.size(0)
+
+            spec = row[1]
+            spec_padded[i, :, :spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wav = row[2]
+            wav_padded[i, :, :wav.size(1)] = wav
+            wav_lengths[i] = wav.size(1)
+
+            sid[i] = row[3]
+
+        if self.return_ids:
+            return (text_padded, text_lengths, spec_padded, spec_lengths,
+                    wav_padded, wav_lengths, sid, ids_sorted_decreasing)
+        return (text_padded, text_lengths, spec_padded, spec_lengths,
+                wav_padded, wav_lengths, sid)
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler
+                               ):
+    """
+    Maintain similar input lengths in a batch.
+    Length groups are specified by boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any batch is included either
+    {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
+
+    It removes samples which are not included in the boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1
+    or length(x) > b3 are discarded.
+    """
+    def __init__(self,
+                 dataset,
+                 batch_size,
+                 boundaries,
+                 num_replicas=None,
+                 rank=None,
+                 shuffle=True):
+        super().__init__(dataset,
+                         num_replicas=num_replicas,
+                         rank=rank,
+                         shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+
+        for i in range(len(buckets) - 1, 0, -1):
+            if len(buckets[i]) == 0:
+                buckets.pop(i)
+                self.boundaries.pop(i + 1)
+
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (total_batch_size -
+                   (len_bucket % total_batch_size)) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+
+    def __iter__(self):
+        # deterministically shuffle based on epoch
+        g = torch.Generator()
+        g.manual_seed(self.epoch)
+
+        indices = []
+        if self.shuffle:
+            for bucket in self.buckets:
+                indices.append(
+                    torch.randperm(len(bucket), generator=g).tolist())
+        else:
+            for bucket in self.buckets:
+                indices.append(list(range(len(bucket))))
+
+        batches = []
+        for i in range(len(self.buckets)):
+            bucket = self.buckets[i]
+            len_bucket = len(bucket)
+            ids_bucket = indices[i]
+            num_samples_bucket = self.num_samples_per_bucket[i]
+
+            # add extra samples to make it evenly divisible
+            rem = num_samples_bucket - len_bucket
+            ids_bucket = ids_bucket + ids_bucket * (
+                rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
+
+            # subsample
+            ids_bucket = ids_bucket[self.rank::self.num_replicas]
+
+            # batching
+            for j in range(len(ids_bucket) // self.batch_size):
+                batch = [
+                    bucket[idx]
+                    for idx in ids_bucket[j * self.batch_size:(j + 1) *
+                                          self.batch_size]
+                ]
+                batches.append(batch)
+
+        if self.shuffle:
+            batch_ids = torch.randperm(len(batches), generator=g).tolist()
+            batches = [batches[i] for i in batch_ids]
+        self.batches = batches
+
+        assert len(self.batches) * self.batch_size == self.num_samples
+        return iter(self.batches)
+
+    def _bisect(self, x, lo=0, hi=None):
+        if hi is None:
+            hi = len(self.boundaries) - 1
+
+        if hi > lo:
+            mid = (hi + lo) // 2
+            if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]:
+                return mid
+            elif x <= self.boundaries[mid]:
+                return self._bisect(x, lo, mid)
+            else:
+                return self._bisect(x, mid + 1, hi)
+        else:
+            return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

export/vits/export_onnx.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import sys
+
+import torch
+
+from models import SynthesizerTrn
+import utils
+
+try:
+    import onnxruntime as ort
+except ImportError:
+    print('Please install onnxruntime!')
+    sys.exit(1)
+
+
+def to_numpy(tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='export onnx model')
+    parser.add_argument('--checkpoint', required=True, help='checkpoint')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--onnx_model', required=True, help='onnx model name')
+    # parser.add_argument('--phone_table',
+    #                     required=True,
+    #                     help='input phone dict')
+    # parser.add_argument('--speaker_table', default=None, help='speaker table')
+    # parser.add_argument("--speaker_num", required=True,
+    #                     type=int, help="speaker num")
+    parser.add_argument(
+        '--providers',
+        required=False,
+        default='CPUExecutionProvider',
+        choices=['CUDAExecutionProvider', 'CPUExecutionProvider'],
+        help='the model to send request to')
+    args = parser.parse_args()
+    return args
+
+
+def get_data_from_cfg(cfg_path: str):
+    assert os.path.isfile(cfg_path)
+    with open(cfg_path, 'r') as f:
+        data = json.load(f)
+        symbols = data["symbols"]
+        speaker_num = data["data"]["n_speakers"]
+    return len(symbols), speaker_num
+
+
+def main():
+    args = get_args()
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+    hps = utils.get_hparams_from_file(args.cfg)
+    # with open(args.phone_table) as p_f:
+    #     phone_num = len(p_f.readlines()) + 1
+    # num_speakers = 1
+    # if args.speaker_table is not None:
+    #     num_speakers = len(open(args.speaker_table).readlines()) + 1
+    phone_num, num_speakers = get_data_from_cfg(args.cfg)
+    net_g = SynthesizerTrn(phone_num,
+                           hps.data.filter_length // 2 + 1,
+                           hps.train.segment_size // hps.data.hop_length,
+                           n_speakers=num_speakers,
+                           **hps.model)
+    utils.load_checkpoint(args.checkpoint, net_g, None)
+    net_g.forward = net_g.export_forward
+    net_g.eval()
+
+    seq = torch.randint(low=0, high=phone_num, size=(1, 10), dtype=torch.long)
+    seq_len = torch.IntTensor([seq.size(1)]).long()
+
+    # noise(可用于控制感情等变化程度) lenth(可用于控制整体语速) noisew(控制音素发音长度变化程度)
+    # 参考 https://github.com/gbxh/genshinTTS
+    scales = torch.FloatTensor([0.667, 1.0, 0.8])
+    # make triton dynamic shape happy
+    scales = scales.unsqueeze(0)
+    sid = torch.IntTensor([0]).long()
+
+    dummy_input = (seq, seq_len, scales, sid)
+    torch.onnx.export(model=net_g,
+                      args=dummy_input,
+                      f=args.onnx_model,
+                      input_names=['input', 'input_lengths', 'scales', 'sid'],
+                      output_names=['output'],
+                      dynamic_axes={
+                          'input': {
+                              0: 'batch',
+                              1: 'phonemes'
+                          },
+                          'input_lengths': {
+                              0: 'batch'
+                          },
+                          'scales': {
+                              0: 'batch'
+                          },
+                          'sid': {
+                              0: 'batch'
+                          },
+                          'output': {
+                              0: 'batch',
+                              1: 'audio',
+                              2: 'audio_length'
+                          }
+                      },
+                      opset_version=13,
+                      verbose=False)
+
+    # Verify onnx precision
+    torch_output = net_g(seq, seq_len, scales, sid)
+    providers = [args.providers]
+    ort_sess = ort.InferenceSession(args.onnx_model, providers=providers)
+    ort_inputs = {
+        'input': to_numpy(seq),
+        'input_lengths': to_numpy(seq_len),
+        'scales': to_numpy(scales),
+        'sid': to_numpy(sid),
+    }
+    onnx_output = ort_sess.run(None, ort_inputs)
+
+
+if __name__ == '__main__':
+    main()

export/vits/inference.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+import numpy as np
+from scipy.io import wavfile
+import torch
+
+import commons
+from models import SynthesizerTrn
+import utils
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='inference')
+    parser.add_argument('--checkpoint', required=True, help='checkpoint')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--outdir', required=True, help='ouput directory')
+    parser.add_argument('--phone_table',
+                        required=True,
+                        help='input phone dict')
+    parser.add_argument('--speaker_table', default=None, help='speaker table')
+    parser.add_argument('--test_file', required=True, help='test file')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    print(args)
+    phone_dict = {}
+    with open(args.phone_table) as p_f:
+        for line in p_f:
+            phone_id = line.strip().split()
+            phone_dict[phone_id[0]] = int(phone_id[1])
+    speaker_dict = {}
+    if args.speaker_table is not None:
+        with open(args.speaker_table) as p_f:
+            for line in p_f:
+                arr = line.strip().split()
+                assert len(arr) == 2
+                speaker_dict[arr[0]] = int(arr[1])
+    hps = utils.get_hparams_from_file(args.cfg)
+
+    net_g = SynthesizerTrn(
+        len(phone_dict) + 1,
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=len(speaker_dict) + 1,  # 0 is kept for unknown speaker
+        **hps.model).cuda()
+    net_g.eval()
+    utils.load_checkpoint(args.checkpoint, net_g, None)
+
+    with open(args.test_file) as fin:
+        for line in fin:
+            arr = line.strip().split("|")
+            audio_path = arr[0]
+            if len(arr) == 2:
+                sid = 0
+                text = arr[1]
+            else:
+                sid = speaker_dict[arr[1]]
+                text = arr[2]
+            seq = [phone_dict[symbol] for symbol in text.split()]
+            if hps.data.add_blank:
+                seq = commons.intersperse(seq, 0)
+            seq = torch.LongTensor(seq)
+            with torch.no_grad():
+                x = seq.cuda().unsqueeze(0)
+                x_length = torch.LongTensor([seq.size(0)]).cuda()
+                sid = torch.LongTensor([sid]).cuda()
+                audio = net_g.infer(
+                    x,
+                    x_length,
+                    sid=sid,
+                    noise_scale=.667,
+                    noise_scale_w=0.8,
+                    length_scale=1)[0][0, 0].data.cpu().float().numpy()
+                audio *= 32767 / max(0.01, np.max(np.abs(audio))) * 0.6
+                audio = np.clip(audio, -32767.0, 32767.0)
+                wavfile.write(args.outdir + "/" + audio_path.split("/")[-1],
+                              hps.data.sampling_rate, audio.astype(np.int16))
+
+
+if __name__ == '__main__':
+    main()

export/vits/inference_onnx.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2022, Yongqiang Li (yongqiangli@alumni.hust.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from text import text_to_sequence
+import numpy as np
+from scipy.io import wavfile
+import torch
+import json
+import commons
+import utils
+import sys
+import pathlib
+
+try:
+    import onnxruntime as ort
+except ImportError:
+    print('Please install onnxruntime!')
+    sys.exit(1)
+
+
+def to_numpy(tensor: torch.Tensor):
+    return tensor.detach().cpu().numpy() if tensor.requires_grad \
+        else tensor.detach().numpy()
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='inference')
+    parser.add_argument('--onnx_model', required=True, help='onnx model')
+    parser.add_argument('--cfg', required=True, help='config file')
+    parser.add_argument('--outdir', default="onnx_output",
+                        help='ouput directory')
+    # parser.add_argument('--phone_table',
+    #                     required=True,
+    #                     help='input phone dict')
+    # parser.add_argument('--speaker_table', default=None, help='speaker table')
+    parser.add_argument('--test_file', required=True, help='test file')
+    args = parser.parse_args()
+    return args
+
+
+def get_symbols_from_json(path):
+    import os
+    assert os.path.isfile(path)
+    with open(path, 'r') as f:
+        data = json.load(f)
+    return data['symbols']
+
+
+def main():
+    args = get_args()
+    print(args)
+    if not pathlib.Path(args.outdir).exists():
+        pathlib.Path(args.outdir).mkdir(exist_ok=True, parents=True)
+    # phones =
+    symbols = get_symbols_from_json(args.cfg)
+    phone_dict = {
+        symbol: i for i, symbol in enumerate(symbols)
+    }
+
+    # speaker_dict = {}
+    # if args.speaker_table is not None:
+    #     with open(args.speaker_table) as p_f:
+    #         for line in p_f:
+    #             arr = line.strip().split()
+    #             assert len(arr) == 2
+    #             speaker_dict[arr[0]] = int(arr[1])
+    hps = utils.get_hparams_from_file(args.cfg)
+
+    ort_sess = ort.InferenceSession(args.onnx_model)
+
+    with open(args.test_file) as fin:
+        for line in fin:
+            arr = line.strip().split("|")
+            audio_path = arr[0]
+        
+            # TODO: 控制说话人编号
+            sid = 8
+            text = arr[1]
+            # else:
+            #     sid = speaker_dict[arr[1]]
+            #     text = arr[2]
+            seq = text_to_sequence(text, symbols=hps.symbols, cleaner_names=["japanese_cleaners2"]
+                                   )
+            if hps.data.add_blank:
+                seq = commons.intersperse(seq, 0)
+
+            # if hps.data.add_blank:
+            #     seq = commons.intersperse(seq, 0)
+            with torch.no_grad():
+                #         x = torch.LongTensor([seq])
+                #         x_len = torch.IntTensor([x.size(1)]).long()
+                #         sid = torch.LongTensor([sid]).long()
+                #         scales = torch.FloatTensor([0.667, 1.0, 1])
+                # # make triton dynamic shape happy
+                #         scales = scales.unsqueeze(0)
+
+                # use numpy to replace torch
+                x = np.array([seq], dtype=np.int64)
+                x_len = np.array([x.shape[1]], dtype=np.int64)
+                sid = np.array([sid], dtype=np.int64)
+                # noise(可用于控制感情等变化程度) lenth(可用于控制整体语速) noisew(控制音素发音长度变化程度)
+                # 参考 https://github.com/gbxh/genshinTTS
+                scales = np.array([0.667, 0.8, 1], dtype=np.float32)
+                # scales = scales[np.newaxis, :]
+                # scales.reshape(1, -1)
+                scales.resize(1, 3)
+
+                ort_inputs = {
+                    'input': x,
+                    'input_lengths': x_len,
+                    'scales': scales,
+                    'sid': sid
+                }
+
+                # ort_inputs = {
+                #     'input': to_numpy(x),
+                #     'input_lengths': to_numpy(x_len),
+                #     'scales': to_numpy(scales),
+                #     'sid': to_numpy(sid)
+                # }
+                import time
+                # start_time = time.time()
+                start_time = time.perf_counter()
+                audio = np.squeeze(ort_sess.run(None, ort_inputs))
+                audio *= 32767.0 / max(0.01, np.max(np.abs(audio))) * 0.6
+                audio = np.clip(audio, -32767.0, 32767.0)
+                end_time = time.perf_counter()
+                # end_time = time.time()
+                print("infer time cost: ", end_time - start_time, "s")
+
+                wavfile.write(args.outdir + "/" + audio_path.split("/")[-1],
+                              hps.data.sampling_rate, audio.astype(np.int16))
+
+
+if __name__ == '__main__':
+    main()

export/vits/losses.py

@@ -0,0 +1,58 @@
+import torch
+
+
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            rl = rl.float().detach()
+            gl = gl.float()
+            loss += torch.mean(torch.abs(rl - gl))
+
+    return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+    r_losses = []
+    g_losses = []
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        dr = dr.float()
+        dg = dg.float()
+        r_loss = torch.mean((1 - dr)**2)
+        g_loss = torch.mean(dg**2)
+        loss += (r_loss + g_loss)
+        r_losses.append(r_loss.item())
+        g_losses.append(g_loss.item())
+
+    return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+    loss = 0
+    gen_losses = []
+    for dg in disc_outputs:
+        dg = dg.float()
+        l = torch.mean((1 - dg)**2)
+        gen_losses.append(l)
+        loss += l
+
+    return loss, gen_losses
+
+
+def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
+    """
+  z_p, logs_q: [b, h, t_t]
+  m_p, logs_p: [b, h, t_t]
+  """
+    z_p = z_p.float()
+    logs_q = logs_q.float()
+    m_p = m_p.float()
+    logs_p = logs_p.float()
+    z_mask = z_mask.float()
+
+    kl = logs_p - logs_q - 0.5
+    kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
+    kl = torch.sum(kl * z_mask)
+    l = kl / torch.sum(z_mask)
+    return l

export/vits/mel_processing.py

@@ -0,0 +1,137 @@
+import torch
+import torch.nn.functional as F
+import torch.utils.data
+from librosa.filters import mel as librosa_mel_fn
+
+MAX_WAV_VALUE = 32768.0
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    """
+    PARAMS
+    ------
+    C: compression factor
+    """
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    """
+    PARAMS
+    ------
+    C: compression factor used to compress
+    """
+    return torch.exp(x) / C
+
+
+def spectral_normalize_torch(magnitudes):
+    output = dynamic_range_compression_torch(magnitudes)
+    return output
+
+
+def spectral_de_normalize_torch(magnitudes):
+    output = dynamic_range_decompression_torch(magnitudes)
+    return output
+
+
+mel_basis = {}
+hann_window = {}
+
+
+def spectrogram_torch(y,
+                      n_fft,
+                      sampling_rate,
+                      hop_size,
+                      win_size,
+                      center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device)
+
+    y = F.pad(y.unsqueeze(1),
+              (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+              mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y,
+                      n_fft,
+                      hop_length=hop_size,
+                      win_length=win_size,
+                      window=hann_window[wnsize_dtype_device],
+                      center=center,
+                      pad_mode='reflect',
+                      normalized=False,
+                      onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+    return spec
+
+
+def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
+    global mel_basis
+    dtype_device = str(spec.dtype) + '_' + str(spec.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=spec.dtype, device=spec.device)
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+    return spec
+
+
+def mel_spectrogram_torch(y,
+                          n_fft,
+                          num_mels,
+                          sampling_rate,
+                          hop_size,
+                          win_size,
+                          fmin,
+                          fmax,
+                          center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global mel_basis, hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=y.dtype, device=y.device)
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device)
+
+    y = F.pad(y.unsqueeze(1),
+              (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+              mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y,
+                      n_fft,
+                      hop_length=hop_size,
+                      win_length=win_size,
+                      window=hann_window[wnsize_dtype_device],
+                      center=center,
+                      pad_mode='reflect',
+                      normalized=False,
+                      onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+
+    return spec

export/vits/models.py

@@ -0,0 +1,672 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.nn import Conv1d, ConvTranspose1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+import monotonic_align
+
+import commons
+import modules
+import attentions
+from commons import init_weights, get_padding
+
+
+class StochasticDurationPredictor(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout,
+                 n_flows=4,
+                 gin_channels=0):
+        super().__init__()
+        filter_channels = in_channels  # it needs to be removed from future version.
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.n_flows = n_flows
+        self.gin_channels = gin_channels
+
+        self.log_flow = modules.Log()
+        self.flows = nn.ModuleList()
+        self.flows.append(modules.ElementwiseAffine(2))
+        for i in range(n_flows):
+            self.flows.append(
+                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
+            self.flows.append(modules.Flip())
+
+        self.post_pre = nn.Conv1d(1, filter_channels, 1)
+        self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
+        self.post_convs = modules.DDSConv(filter_channels,
+                                          kernel_size,
+                                          n_layers=3,
+                                          p_dropout=p_dropout)
+        self.post_flows = nn.ModuleList()
+        self.post_flows.append(modules.ElementwiseAffine(2))
+        for i in range(4):
+            self.post_flows.append(
+                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
+            self.post_flows.append(modules.Flip())
+
+        self.pre = nn.Conv1d(in_channels, filter_channels, 1)
+        self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
+        self.convs = modules.DDSConv(filter_channels,
+                                     kernel_size,
+                                     n_layers=3,
+                                     p_dropout=p_dropout)
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
+
+    def forward(self,
+                x,
+                x_mask,
+                w=None,
+                g=None,
+                reverse=False,
+                noise_scale=1.0):
+        x = torch.detach(x)
+        x = self.pre(x)
+        if g is not None:
+            g = torch.detach(g)
+            x = x + self.cond(g)
+        x = self.convs(x, x_mask)
+        x = self.proj(x) * x_mask
+
+        if not reverse:
+            flows = self.flows
+            assert w is not None
+
+            logdet_tot_q = 0
+            h_w = self.post_pre(w)
+            h_w = self.post_convs(h_w, x_mask)
+            h_w = self.post_proj(h_w) * x_mask
+            e_q = torch.randn(w.size(0), 2, w.size(2)).to(
+                device=x.device, dtype=x.dtype) * x_mask
+            z_q = e_q
+            for flow in self.post_flows:
+                z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
+                logdet_tot_q += logdet_q
+            z_u, z1 = torch.split(z_q, [1, 1], 1)
+            u = torch.sigmoid(z_u) * x_mask
+            z0 = (w - u) * x_mask
+            logdet_tot_q += torch.sum(
+                (F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2])
+            logq = torch.sum(
+                -0.5 * (math.log(2 * math.pi) +
+                        (e_q**2)) * x_mask, [1, 2]) - logdet_tot_q
+
+            logdet_tot = 0
+            z0, logdet = self.log_flow(z0, x_mask)
+            logdet_tot += logdet
+            z = torch.cat([z0, z1], 1)
+            for flow in flows:
+                z, logdet = flow(z, x_mask, g=x, reverse=reverse)
+                logdet_tot = logdet_tot + logdet
+            nll = torch.sum(0.5 * (math.log(2 * math.pi) +
+                                   (z**2)) * x_mask, [1, 2]) - logdet_tot
+            return nll + logq  # [b]
+        else:
+            flows = list(reversed(self.flows))
+            flows = flows[:-2] + [flows[-1]]  # remove a useless vflow
+            z = torch.randn(x.size(0), 2, x.size(2)).to(
+                device=x.device, dtype=x.dtype) * noise_scale
+            for flow in flows:
+                z = flow(z, x_mask, g=x, reverse=reverse)
+            z0, z1 = torch.split(z, [1, 1], 1)
+            logw = z0
+            return logw
+
+
+class DurationPredictor(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 p_dropout,
+                 gin_channels=0):
+        super().__init__()
+
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.gin_channels = gin_channels
+
+        self.drop = nn.Dropout(p_dropout)
+        self.conv_1 = nn.Conv1d(in_channels,
+                                filter_channels,
+                                kernel_size,
+                                padding=kernel_size // 2)
+        self.norm_1 = modules.LayerNorm(filter_channels)
+        self.conv_2 = nn.Conv1d(filter_channels,
+                                filter_channels,
+                                kernel_size,
+                                padding=kernel_size // 2)
+        self.norm_2 = modules.LayerNorm(filter_channels)
+        self.proj = nn.Conv1d(filter_channels, 1, 1)
+
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, in_channels, 1)
+
+    def forward(self, x, x_mask, g=None):
+        x = torch.detach(x)
+        if g is not None:
+            g = torch.detach(g)
+            x = x + self.cond(g)
+        x = self.conv_1(x * x_mask)
+        x = torch.relu(x)
+        x = self.norm_1(x)
+        x = self.drop(x)
+        x = self.conv_2(x * x_mask)
+        x = torch.relu(x)
+        x = self.norm_2(x)
+        x = self.drop(x)
+        x = self.proj(x * x_mask)
+        return x * x_mask
+
+
+class TextEncoder(nn.Module):
+    def __init__(self, n_vocab, out_channels, hidden_channels, filter_channels,
+                 n_heads, n_layers, kernel_size, p_dropout):
+        super().__init__()
+        self.n_vocab = n_vocab
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+
+        self.emb = nn.Embedding(n_vocab, hidden_channels)
+        nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
+
+        self.encoder = attentions.Encoder(hidden_channels, filter_channels,
+                                          n_heads, n_layers, kernel_size,
+                                          p_dropout)
+        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(self, x, x_lengths):
+        x = self.emb(x) * math.sqrt(self.hidden_channels)  # [b, t, h]
+        x = torch.transpose(x, 1, -1)  # [b, h, t]
+        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
+                                 1).to(x.dtype)
+
+        x = self.encoder(x * x_mask, x_mask)
+        stats = self.proj(x) * x_mask
+
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        return x, m, logs, x_mask
+
+
+class ResidualCouplingBlock(nn.Module):
+    def __init__(self,
+                 channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 n_flows=4,
+                 gin_channels=0):
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.n_flows = n_flows
+        self.gin_channels = gin_channels
+
+        self.flows = nn.ModuleList()
+        for i in range(n_flows):
+            self.flows.append(
+                modules.ResidualCouplingLayer(channels,
+                                              hidden_channels,
+                                              kernel_size,
+                                              dilation_rate,
+                                              n_layers,
+                                              gin_channels=gin_channels,
+                                              mean_only=True))
+            self.flows.append(modules.Flip())
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        if not reverse:
+            for flow in self.flows:
+                x, _ = flow(x, x_mask, g=g, reverse=reverse)
+        else:
+            for flow in reversed(self.flows):
+                x = flow(x, x_mask, g=g, reverse=reverse)
+        return x
+
+
+class PosteriorEncoder(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 gin_channels=0):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+
+        self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
+        self.enc = modules.WN(hidden_channels,
+                              kernel_size,
+                              dilation_rate,
+                              n_layers,
+                              gin_channels=gin_channels)
+        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(self, x, x_lengths, g=None):
+        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
+                                 1).to(x.dtype)
+        x = self.pre(x) * x_mask
+        x = self.enc(x, x_mask, g=g)
+        stats = self.proj(x) * x_mask
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+        return z, m, logs, x_mask
+
+
+class Generator(torch.nn.Module):
+    def __init__(self,
+                 initial_channel,
+                 resblock,
+                 resblock_kernel_sizes,
+                 resblock_dilation_sizes,
+                 upsample_rates,
+                 upsample_initial_channel,
+                 upsample_kernel_sizes,
+                 gin_channels=0):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        self.conv_pre = Conv1d(initial_channel,
+                               upsample_initial_channel,
+                               7,
+                               1,
+                               padding=3)
+        resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(
+                weight_norm(
+                    ConvTranspose1d(upsample_initial_channel // (2**i),
+                                    upsample_initial_channel // (2**(i + 1)),
+                                    k,
+                                    u,
+                                    padding=(k - u) // 2)))
+
+        self.resblocks = nn.ModuleList()
+        for i in range(len(self.ups)):
+            ch = upsample_initial_channel // (2**(i + 1))
+            for j, (k, d) in enumerate(
+                    zip(resblock_kernel_sizes, resblock_dilation_sizes)):
+                self.resblocks.append(resblock(ch, k, d))
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+    def forward(self, x, g=None):
+        x = self.conv_pre(x)
+        if g is not None:
+            x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            x = self.ups[i](x)
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs / self.num_kernels
+        x = F.leaky_relu(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def remove_weight_norm(self):
+        print('Removing weight norm...')
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self,
+                 period,
+                 kernel_size=5,
+                 stride=3,
+                 use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        self.use_spectral_norm = use_spectral_norm
+        norm_f = weight_norm if use_spectral_norm is False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(
+                Conv2d(1,
+                       32, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(32,
+                       128, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(128,
+                       512, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(512,
+                       1024, (kernel_size, 1), (stride, 1),
+                       padding=(get_padding(kernel_size, 1), 0))),
+            norm_f(
+                Conv2d(1024,
+                       1024, (kernel_size, 1),
+                       1,
+                       padding=(get_padding(kernel_size, 1), 0))),
+        ])
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0:  # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class DiscriminatorS(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(DiscriminatorS, self).__init__()
+        norm_f = weight_norm if use_spectral_norm is False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv1d(1, 16, 15, 1, padding=7)),
+            norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
+            norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
+            norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
+            norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
+            norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
+        ])
+        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
+
+    def forward(self, x):
+        fmap = []
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(MultiPeriodDiscriminator, self).__init__()
+        periods = [2, 3, 5, 7, 11]
+
+        discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
+        discs = discs + [
+            DiscriminatorP(i, use_spectral_norm=use_spectral_norm)
+            for i in periods
+        ]
+        self.discriminators = nn.ModuleList(discs)
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+class SynthesizerTrn(nn.Module):
+    """
+  Synthesizer for Training
+  """
+    def __init__(self,
+                 n_vocab,
+                 spec_channels,
+                 segment_size,
+                 inter_channels,
+                 hidden_channels,
+                 filter_channels,
+                 n_heads,
+                 n_layers,
+                 kernel_size,
+                 p_dropout,
+                 resblock,
+                 resblock_kernel_sizes,
+                 resblock_dilation_sizes,
+                 upsample_rates,
+                 upsample_initial_channel,
+                 upsample_kernel_sizes,
+                 n_speakers=0,
+                 gin_channels=0,
+                 use_sdp=True,
+                 **kwargs):
+
+        super().__init__()
+        self.n_vocab = n_vocab
+        self.spec_channels = spec_channels
+        self.inter_channels = inter_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.resblock = resblock
+        self.resblock_kernel_sizes = resblock_kernel_sizes
+        self.resblock_dilation_sizes = resblock_dilation_sizes
+        self.upsample_rates = upsample_rates
+        self.upsample_initial_channel = upsample_initial_channel
+        self.upsample_kernel_sizes = upsample_kernel_sizes
+        self.segment_size = segment_size
+        self.n_speakers = n_speakers
+        self.gin_channels = gin_channels
+        if self.n_speakers != 0:
+            message = "gin_channels must be none zero for multiple speakers"
+            assert gin_channels != 0, message
+
+        self.use_sdp = use_sdp
+
+        self.enc_p = TextEncoder(n_vocab, inter_channels, hidden_channels,
+                                 filter_channels, n_heads, n_layers,
+                                 kernel_size, p_dropout)
+        self.dec = Generator(inter_channels,
+                             resblock,
+                             resblock_kernel_sizes,
+                             resblock_dilation_sizes,
+                             upsample_rates,
+                             upsample_initial_channel,
+                             upsample_kernel_sizes,
+                             gin_channels=gin_channels)
+        self.enc_q = PosteriorEncoder(spec_channels,
+                                      inter_channels,
+                                      hidden_channels,
+                                      5,
+                                      1,
+                                      16,
+                                      gin_channels=gin_channels)
+        self.flow = ResidualCouplingBlock(inter_channels,
+                                          hidden_channels,
+                                          5,
+                                          1,
+                                          4,
+                                          gin_channels=gin_channels)
+
+        if use_sdp:
+            self.dp = StochasticDurationPredictor(hidden_channels,
+                                                  192,
+                                                  3,
+                                                  0.5,
+                                                  4,
+                                                  gin_channels=gin_channels)
+        else:
+            self.dp = DurationPredictor(hidden_channels,
+                                        256,
+                                        3,
+                                        0.5,
+                                        gin_channels=gin_channels)
+
+        if n_speakers > 1:
+            self.emb_g = nn.Embedding(n_speakers, gin_channels)
+
+    def forward(self, x, x_lengths, y, y_lengths, sid=None):
+
+        x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
+        if self.n_speakers > 0:
+            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
+        else:
+            g = None
+
+        z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
+        z_p = self.flow(z, y_mask, g=g)
+
+        with torch.no_grad():
+            # negative cross-entropy
+            s_p_sq_r = torch.exp(-2 * logs_p)  # [b, d, t]
+            neg_cent1 = torch.sum(-0.5 * math.log(2 * math.pi) - logs_p, [1],
+                                  keepdim=True)  # [b, 1, t_s]
+            neg_cent2 = torch.matmul(
+                -0.5 * (z_p**2).transpose(1, 2),
+                s_p_sq_r)  # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
+            neg_cent3 = torch.matmul(
+                z_p.transpose(1, 2),
+                (m_p * s_p_sq_r))  # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
+            neg_cent4 = torch.sum(-0.5 * (m_p**2) * s_p_sq_r, [1],
+                                  keepdim=True)  # [b, 1, t_s]
+            neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
+
+            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(
+                y_mask, -1)
+            attn = monotonic_align.maximum_path(
+                neg_cent, attn_mask.squeeze(1)).unsqueeze(1).detach()
+
+        w = attn.sum(2)
+        if self.use_sdp:
+            l_length = self.dp(x, x_mask, w, g=g)
+            l_length = l_length / torch.sum(x_mask)
+        else:
+            logw_ = torch.log(w + 1e-6) * x_mask
+            logw = self.dp(x, x_mask, g=g)
+            l_length = torch.sum(
+                (logw - logw_)**2, [1, 2]) / torch.sum(x_mask)  # for averaging
+
+        # expand prior
+        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1,
+                                                          2)).transpose(1, 2)
+        logs_p = torch.matmul(attn.squeeze(1),
+                              logs_p.transpose(1, 2)).transpose(1, 2)
+
+        z_slice, ids_slice = commons.rand_slice_segments(
+            z, y_lengths, self.segment_size)
+        o = self.dec(z_slice, g=g)
+        return o, l_length, attn, ids_slice, x_mask, y_mask, (z, z_p, m_p,
+                                                              logs_p, m_q,
+                                                              logs_q)
+
+    def infer(self,
+              x,
+              x_lengths,
+              sid=None,
+              noise_scale=1,
+              length_scale=1,
+              noise_scale_w=1.,
+              max_len=None):
+        x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
+        if self.n_speakers > 0:
+            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
+        else:
+            g = None
+
+        if self.use_sdp:
+            logw = self.dp(x,
+                           x_mask,
+                           g=g,
+                           reverse=True,
+                           noise_scale=noise_scale_w)
+        else:
+            logw = self.dp(x, x_mask, g=g)
+        w = torch.exp(logw) * x_mask * length_scale
+        w_ceil = torch.ceil(w)
+        y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
+        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None),
+                                 1).to(x_mask.dtype)
+        attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
+        attn = commons.generate_path(w_ceil, attn_mask)
+
+        m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(
+            1, 2)  # [b, t', t], [b, t, d] -> [b, d, t']
+        logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(
+            1, 2)).transpose(1, 2)  # [b, t', t], [b, t, d] -> [b, d, t']
+
+        z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
+        z = self.flow(z_p, y_mask, g=g, reverse=True)
+        o = self.dec((z * y_mask)[:, :, :max_len], g=g)
+        return o, attn, y_mask, (z, z_p, m_p, logs_p)
+
+    def export_forward(self, x, x_lengths, scales, sid):
+        # shape of scales: Bx3, make triton happy
+        audio, *_ = self.infer(x,
+                               x_lengths,
+                               sid,
+                               noise_scale=scales[0][0],
+                               length_scale=scales[0][1],
+                               noise_scale_w=scales[0][2])
+        return audio
+
+    def voice_conversion(self, y, y_lengths, sid_src, sid_tgt):
+        assert self.n_speakers > 0, "n_speakers have to be larger than 0."
+        g_src = self.emb_g(sid_src).unsqueeze(-1)
+        g_tgt = self.emb_g(sid_tgt).unsqueeze(-1)
+        z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g_src)
+        z_p = self.flow(z, y_mask, g=g_src)
+        z_hat = self.flow(z_p, y_mask, g=g_tgt, reverse=True)
+        o_hat = self.dec(z_hat * y_mask, g=g_tgt)
+        return o_hat, y_mask, (z, z_p, z_hat)

export/vits/modules.py

@@ -0,0 +1,469 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.nn import Conv1d
+from torch.nn.utils import weight_norm, remove_weight_norm
+
+import commons
+from commons import init_weights, get_padding
+from transforms import piecewise_rational_quadratic_transform
+
+LRELU_SLOPE = 0.1
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels, ), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+
+
+class ConvReluNorm(nn.Module):
+    def __init__(self, in_channels, hidden_channels, out_channels, kernel_size,
+                 n_layers, p_dropout):
+        super().__init__()
+        self.in_channels = in_channels
+        self.hidden_channels = hidden_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+        assert n_layers > 1, "Number of layers should be larger than 0."
+
+        self.conv_layers = nn.ModuleList()
+        self.norm_layers = nn.ModuleList()
+        self.conv_layers.append(
+            nn.Conv1d(in_channels,
+                      hidden_channels,
+                      kernel_size,
+                      padding=kernel_size // 2))
+        self.norm_layers.append(LayerNorm(hidden_channels))
+        self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
+        for _ in range(n_layers - 1):
+            self.conv_layers.append(
+                nn.Conv1d(hidden_channels,
+                          hidden_channels,
+                          kernel_size,
+                          padding=kernel_size // 2))
+            self.norm_layers.append(LayerNorm(hidden_channels))
+        self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask):
+        x_org = x
+        for i in range(self.n_layers):
+            x = self.conv_layers[i](x * x_mask)
+            x = self.norm_layers[i](x)
+            x = self.relu_drop(x)
+        x = x_org + self.proj(x)
+        return x * x_mask
+
+
+class DDSConv(nn.Module):
+    """
+  Dialted and Depth-Separable Convolution
+  """
+    def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
+        super().__init__()
+        self.channels = channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+
+        self.drop = nn.Dropout(p_dropout)
+        self.convs_sep = nn.ModuleList()
+        self.convs_1x1 = nn.ModuleList()
+        self.norms_1 = nn.ModuleList()
+        self.norms_2 = nn.ModuleList()
+        for i in range(n_layers):
+            dilation = kernel_size**i
+            padding = (kernel_size * dilation - dilation) // 2
+            self.convs_sep.append(
+                nn.Conv1d(channels,
+                          channels,
+                          kernel_size,
+                          groups=channels,
+                          dilation=dilation,
+                          padding=padding))
+            self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
+            self.norms_1.append(LayerNorm(channels))
+            self.norms_2.append(LayerNorm(channels))
+
+    def forward(self, x, x_mask, g=None):
+        if g is not None:
+            x = x + g
+        for i in range(self.n_layers):
+            y = self.convs_sep[i](x * x_mask)
+            y = self.norms_1[i](y)
+            y = F.gelu(y)
+            y = self.convs_1x1[i](y)
+            y = self.norms_2[i](y)
+            y = F.gelu(y)
+            y = self.drop(y)
+            x = x + y
+        return x * x_mask
+
+
+class WN(torch.nn.Module):
+    def __init__(self,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 gin_channels=0,
+                 p_dropout=0):
+        super(WN, self).__init__()
+        assert (kernel_size % 2 == 1)
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size,
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+        self.p_dropout = p_dropout
+
+        self.in_layers = torch.nn.ModuleList()
+        self.res_skip_layers = torch.nn.ModuleList()
+        self.drop = nn.Dropout(p_dropout)
+
+        if gin_channels != 0:
+            cond_layer = torch.nn.Conv1d(gin_channels,
+                                         2 * hidden_channels * n_layers, 1)
+            self.cond_layer = torch.nn.utils.weight_norm(cond_layer,
+                                                         name='weight')
+
+        for i in range(n_layers):
+            dilation = dilation_rate**i
+            padding = int((kernel_size * dilation - dilation) / 2)
+            in_layer = torch.nn.Conv1d(hidden_channels,
+                                       2 * hidden_channels,
+                                       kernel_size,
+                                       dilation=dilation,
+                                       padding=padding)
+            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
+            self.in_layers.append(in_layer)
+
+            # last one is not necessary
+            if i < n_layers - 1:
+                res_skip_channels = 2 * hidden_channels
+            else:
+                res_skip_channels = hidden_channels
+
+            res_skip_layer = torch.nn.Conv1d(hidden_channels,
+                                             res_skip_channels, 1)
+            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer,
+                                                        name='weight')
+            self.res_skip_layers.append(res_skip_layer)
+
+    def forward(self, x, x_mask, g=None, **kwargs):
+        output = torch.zeros_like(x)
+        n_channels_tensor = torch.IntTensor([self.hidden_channels])
+
+        if g is not None:
+            g = self.cond_layer(g)
+
+        for i in range(self.n_layers):
+            x_in = self.in_layers[i](x)
+            if g is not None:
+                cond_offset = i * 2 * self.hidden_channels
+                g_l = g[:,
+                        cond_offset:cond_offset + 2 * self.hidden_channels, :]
+            else:
+                g_l = torch.zeros_like(x_in)
+
+            acts = commons.fused_add_tanh_sigmoid_multiply(
+                x_in, g_l, n_channels_tensor)
+            acts = self.drop(acts)
+
+            res_skip_acts = self.res_skip_layers[i](acts)
+            if i < self.n_layers - 1:
+                res_acts = res_skip_acts[:, :self.hidden_channels, :]
+                x = (x + res_acts) * x_mask
+                output = output + res_skip_acts[:, self.hidden_channels:, :]
+            else:
+                output = output + res_skip_acts
+        return output * x_mask
+
+    def remove_weight_norm(self):
+        if self.gin_channels != 0:
+            torch.nn.utils.remove_weight_norm(self.cond_layer)
+        for l in self.in_layers:
+            torch.nn.utils.remove_weight_norm(l)
+        for l in self.res_skip_layers:
+            torch.nn.utils.remove_weight_norm(l)
+
+
+class ResBlock1(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super(ResBlock1, self).__init__()
+        self.convs1 = nn.ModuleList([
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[0],
+                       padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[1],
+                       padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[2],
+                       padding=get_padding(kernel_size, dilation[2])))
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=1,
+                       padding=get_padding(kernel_size, 1)))
+        ])
+        self.convs2.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c1, c2 in zip(self.convs1, self.convs2):
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c1(xt)
+            xt = F.leaky_relu(xt, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c2(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+        super(ResBlock2, self).__init__()
+        self.convs = nn.ModuleList([
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[0],
+                       padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(
+                Conv1d(channels,
+                       channels,
+                       kernel_size,
+                       1,
+                       dilation=dilation[1],
+                       padding=get_padding(kernel_size, dilation[1])))
+        ])
+        self.convs.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c in self.convs:
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs:
+            remove_weight_norm(l)
+
+
+class Log(nn.Module):
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
+            logdet = torch.sum(-y, [1, 2])
+            return y, logdet
+        else:
+            x = torch.exp(x) * x_mask
+            return x
+
+
+class Flip(nn.Module):
+    def forward(self, x, *args, reverse=False, **kwargs):
+        x = torch.flip(x, [1])
+        if not reverse:
+            logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+            return x, logdet
+        else:
+            return x
+
+
+class ElementwiseAffine(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.channels = channels
+        self.m = nn.Parameter(torch.zeros(channels, 1))
+        self.logs = nn.Parameter(torch.zeros(channels, 1))
+
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = self.m + torch.exp(self.logs) * x
+            y = y * x_mask
+            logdet = torch.sum(self.logs * x_mask, [1, 2])
+            return y, logdet
+        else:
+            x = (x - self.m) * torch.exp(-self.logs) * x_mask
+            return x
+
+
+class ResidualCouplingLayer(nn.Module):
+    def __init__(self,
+                 channels,
+                 hidden_channels,
+                 kernel_size,
+                 dilation_rate,
+                 n_layers,
+                 p_dropout=0,
+                 gin_channels=0,
+                 mean_only=False):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+        self.enc = WN(hidden_channels,
+                      kernel_size,
+                      dilation_rate,
+                      n_layers,
+                      p_dropout=p_dropout,
+                      gin_channels=gin_channels)
+        self.post = nn.Conv1d(hidden_channels,
+                              self.half_channels * (2 - mean_only), 1)
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+        h = self.enc(h, x_mask, g=g)
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
+
+
+class ConvFlow(nn.Module):
+    def __init__(self,
+                 in_channels,
+                 filter_channels,
+                 kernel_size,
+                 n_layers,
+                 num_bins=10,
+                 tail_bound=5.0):
+        super().__init__()
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.num_bins = num_bins
+        self.tail_bound = tail_bound
+        self.half_channels = in_channels // 2
+
+        self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
+        self.convs = DDSConv(filter_channels,
+                             kernel_size,
+                             n_layers,
+                             p_dropout=0.)
+        self.proj = nn.Conv1d(filter_channels,
+                              self.half_channels * (num_bins * 3 - 1), 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0)
+        h = self.convs(h, x_mask, g=g)
+        h = self.proj(h) * x_mask
+
+        b, c, t = x0.shape
+        h = h.reshape(b, c, -1, t).permute(0, 1, 3,
+                                           2)  # [b, cx?, t] -> [b, c, t, ?]
+
+        unnormalized_widths = h[..., :self.num_bins] / math.sqrt(
+            self.filter_channels)
+        unnormalized_heights = h[...,
+                                 self.num_bins:2 * self.num_bins] / math.sqrt(
+                                     self.filter_channels)
+        unnormalized_derivatives = h[..., 2 * self.num_bins:]
+
+        x1, logabsdet = piecewise_rational_quadratic_transform(
+            x1,
+            unnormalized_widths,
+            unnormalized_heights,
+            unnormalized_derivatives,
+            inverse=reverse,
+            tails='linear',
+            tail_bound=self.tail_bound)
+
+        x = torch.cat([x0, x1], 1) * x_mask
+        logdet = torch.sum(logabsdet * x_mask, [1, 2])
+        if not reverse:
+            return x, logdet
+        else:
+            return x

export/vits/text/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Keith Ito
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

export/vits/text/__init__.py

@@ -0,0 +1,37 @@
+""" from https://github.com/keithito/tacotron """
+from text import cleaners
+
+
+def text_to_sequence(text, symbols, cleaner_names):
+    '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+      Args:
+        text: string to convert to a sequence
+        cleaner_names: names of the cleaner functions to run the text through
+      Returns:
+        List of integers corresponding to the symbols in the text
+    '''
+    _symbol_to_id = {s: i for i, s in enumerate(symbols)}
+
+    sequence = []
+
+    clean_text = _clean_text(text, cleaner_names)
+
+    sequence = [
+        _symbol_to_id[symbol] for symbol in clean_text if symbol in _symbol_to_id.keys()
+    ]
+
+    # for symbol in clean_text:
+    #     if symbol not in _symbol_to_id.keys():
+    #         continue
+    #     symbol_id = _symbol_to_id[symbol]
+    #     sequence += [symbol_id]
+    return sequence
+
+
+def _clean_text(text, cleaner_names):
+    for name in cleaner_names:
+        cleaner = getattr(cleaners, name)
+        if not cleaner:
+            raise Exception('Unknown cleaner: %s' % name)
+        text = cleaner(text)
+    return text

export/vits/text/cleaners.py

@@ -0,0 +1,58 @@
+import re
+from unidecode import unidecode
+import pyopenjtalk
+
+pyopenjtalk._lazy_init()
+
+# Regular expression matching Japanese without punctuation marks:
+_japanese_characters = re.compile(
+    r'[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
+
+# Regular expression matching non-Japanese characters or punctuation marks:
+_japanese_marks = re.compile(
+    r'[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
+
+
+def japanese_cleaners(text):
+    '''Pipeline for notating accent in Japanese text.'''
+    '''Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html'''
+    sentences = re.split(_japanese_marks, text)
+    marks = re.findall(_japanese_marks, text)
+    text = ''
+    for i, sentence in enumerate(sentences):
+        if re.match(_japanese_characters, sentence):
+            if text != '':
+                text += ' '
+            labels = pyopenjtalk.extract_fullcontext(sentence)
+            for n, label in enumerate(labels):
+                phoneme = re.search(r'\-([^\+]*)\+', label).group(1)
+                if phoneme not in ['sil', 'pau']:
+                    text += phoneme.replace('ch', 'ʧ').replace('sh', 'ʃ').replace('cl', 'Q')
+                else:
+                    continue
+                n_moras = int(re.search(r'/F:(\d+)_', label).group(1))
+                a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
+                a2 = int(re.search(r"\+(\d+)\+", label).group(1))
+                a3 = int(re.search(r"\+(\d+)/", label).group(1))
+                if re.search(r'\-([^\+]*)\+', labels[n + 1]).group(1) in ['sil', 'pau']:
+                    a2_next = -1
+                else:
+                    a2_next = int(re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
+                # Accent phrase boundary
+                if a3 == 1 and a2_next == 1:
+                    text += ' '
+                # Falling
+                elif a1 == 0 and a2_next == a2 + 1 and a2 != n_moras:
+                    text += '↓'
+                # Rising
+                elif a2 == 1 and a2_next == 2:
+                    text += '↑'
+        if i < len(marks):
+            text += unidecode(marks[i]).replace(' ', '')
+    if re.match('[A-Za-z]', text[-1]):
+        text += '.'
+    return text
+
+
+def japanese_cleaners2(text):
+    return japanese_cleaners(text).replace('ts','ʦ').replace('...','…')

export/vits/text/symbols.py

@@ -0,0 +1,39 @@
+'''
+Defines the set of symbols used in text input to the model.
+'''
+
+'''# japanese_cleaners
+_pad        = '_'
+_punctuation = ',.!?-'
+_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
+'''
+# jp_cleaners
+_pad        = '_'
+_punctuation = ',.!?-'
+_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
+
+
+
+# japanese_cleaners2
+# _pad        = '_'
+# _punctuation = ',.!?-~…'
+# _letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
+
+
+'''# korean_cleaners
+_pad        = '_'
+_punctuation = ',.!?…~'
+_letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
+'''
+
+'''# chinese_cleaners
+_pad        = '_'
+_punctuation = '，。！？—…'
+_letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
+'''
+
+# Export all symbols:
+symbols = [_pad] + list(_punctuation) + list(_letters)
+
+# Special symbol ids
+SPACE_ID = symbols.index(" ")

export/vits/train.py

@@ -0,0 +1,328 @@
+import os
+
+import torch
+from torch.nn import functional as F
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+import torch.multiprocessing as mp
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.cuda.amp import autocast, GradScaler
+
+import commons
+import utils
+from data_utils import (TextAudioSpeakerLoader, TextAudioSpeakerCollate,
+                        DistributedBucketSampler)
+from models import (
+    SynthesizerTrn,
+    MultiPeriodDiscriminator,
+)
+from losses import (generator_loss, discriminator_loss, feature_loss, kl_loss)
+from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
+
+torch.backends.cudnn.benchmark = True
+global_step = 0
+
+
+def main():
+    """Assume Single Node Multi GPUs Training Only"""
+    assert torch.cuda.is_available(), "CPU training is not allowed."
+
+    n_gpus = torch.cuda.device_count()
+    hps = utils.get_hparams()
+    mp.spawn(run, nprocs=n_gpus, args=(
+        n_gpus,
+        hps,
+    ))
+
+
+def run(rank, n_gpus, hps):
+    global global_step
+    if rank == 0:
+        logger = utils.get_logger(hps.model_dir)
+        logger.info(hps)
+        utils.check_git_hash(hps.model_dir)
+        writer = SummaryWriter(log_dir=hps.model_dir)
+        writer_eval = SummaryWriter(
+            log_dir=os.path.join(hps.model_dir, "eval"))
+
+    dist.init_process_group(backend='nccl',
+                            init_method='env://',
+                            world_size=n_gpus,
+                            rank=rank)
+    torch.manual_seed(hps.train.seed)
+    torch.cuda.set_device(rank)
+    train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps.data)
+    train_sampler = DistributedBucketSampler(
+        train_dataset,
+        hps.train.batch_size, [32, 300, 400, 500, 600, 700, 800, 900, 1000],
+        num_replicas=n_gpus,
+        rank=rank,
+        shuffle=True)
+    collate_fn = TextAudioSpeakerCollate()
+    train_loader = DataLoader(train_dataset,
+                              num_workers=8,
+                              shuffle=False,
+                              pin_memory=True,
+                              collate_fn=collate_fn,
+                              batch_sampler=train_sampler)
+    if rank == 0:
+        eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files,
+                                              hps.data)
+        eval_loader = DataLoader(eval_dataset,
+                                 num_workers=8,
+                                 shuffle=False,
+                                 batch_size=hps.train.batch_size,
+                                 pin_memory=True,
+                                 drop_last=False,
+                                 collate_fn=collate_fn)
+
+    net_g = SynthesizerTrn(hps.data.num_phones,
+                           hps.data.filter_length // 2 + 1,
+                           hps.train.segment_size // hps.data.hop_length,
+                           n_speakers=hps.data.n_speakers,
+                           **hps.model).cuda(rank)
+    net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
+    optim_g = torch.optim.AdamW(net_g.parameters(),
+                                hps.train.learning_rate,
+                                betas=hps.train.betas,
+                                eps=hps.train.eps)
+    optim_d = torch.optim.AdamW(net_d.parameters(),
+                                hps.train.learning_rate,
+                                betas=hps.train.betas,
+                                eps=hps.train.eps)
+    net_g = DDP(net_g, device_ids=[rank])
+    net_d = DDP(net_d, device_ids=[rank])
+
+    try:
+        _, _, _, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g,
+            optim_g)
+        _, _, _, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d,
+            optim_d)
+        global_step = (epoch_str - 1) * len(train_loader)
+    except Exception as e:
+        epoch_str = 1
+        global_step = 0
+
+    scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
+        optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2)
+    scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
+        optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2)
+
+    scaler = GradScaler(enabled=hps.train.fp16_run)
+
+    for epoch in range(epoch_str, hps.train.epochs + 1):
+        if rank == 0:
+            train_and_evaluate(rank, epoch, hps, [net_g, net_d],
+                               [optim_g, optim_d], [scheduler_g, scheduler_d],
+                               scaler, [train_loader, eval_loader], logger,
+                               [writer, writer_eval])
+        else:
+            train_and_evaluate(rank, epoch, hps, [net_g, net_d],
+                               [optim_g, optim_d], [scheduler_g, scheduler_d],
+                               scaler, [train_loader, None], None, None)
+        scheduler_g.step()
+        scheduler_d.step()
+
+
+def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler,
+                       loaders, logger, writers):
+    net_g, net_d = nets
+    optim_g, optim_d = optims
+    scheduler_g, scheduler_d = schedulers
+    train_loader, eval_loader = loaders
+    if writers is not None:
+        writer, writer_eval = writers
+
+    train_loader.batch_sampler.set_epoch(epoch)
+    global global_step
+
+    net_g.train()
+    net_d.train()
+    for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths,
+                    speakers) in enumerate(train_loader):
+        x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(
+            rank, non_blocking=True)
+        spec, spec_lengths = spec.cuda(
+            rank, non_blocking=True), spec_lengths.cuda(rank,
+                                                        non_blocking=True)
+        y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(
+            rank, non_blocking=True)
+        speakers = speakers.cuda(rank, non_blocking=True)
+
+        with autocast(enabled=hps.train.fp16_run):
+            y_hat, l_length, attn, ids_slice, x_mask, z_mask, (
+                z, z_p, m_p, logs_p, m_q,
+                logs_q) = net_g(x, x_lengths, spec, spec_lengths, speakers)
+
+            mel = spec_to_mel_torch(spec, hps.data.filter_length,
+                                    hps.data.n_mel_channels,
+                                    hps.data.sampling_rate, hps.data.mel_fmin,
+                                    hps.data.mel_fmax)
+            y_mel = commons.slice_segments(
+                mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
+            y_hat_mel = mel_spectrogram_torch(
+                y_hat.squeeze(1), hps.data.filter_length,
+                hps.data.n_mel_channels, hps.data.sampling_rate,
+                hps.data.hop_length, hps.data.win_length, hps.data.mel_fmin,
+                hps.data.mel_fmax)
+
+            y = commons.slice_segments(y, ids_slice * hps.data.hop_length,
+                                       hps.train.segment_size)  # slice
+
+            # Discriminator
+            y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
+            with autocast(enabled=False):
+                loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
+                    y_d_hat_r, y_d_hat_g)
+                loss_disc_all = loss_disc
+        optim_d.zero_grad()
+        scaler.scale(loss_disc_all).backward()
+        scaler.unscale_(optim_d)
+        grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
+        scaler.step(optim_d)
+
+        with autocast(enabled=hps.train.fp16_run):
+            # Generator
+            y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
+            with autocast(enabled=False):
+                loss_dur = torch.sum(l_length.float())
+                loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
+                loss_kl = kl_loss(z_p, logs_q, m_p, logs_p,
+                                  z_mask) * hps.train.c_kl
+
+                loss_fm = feature_loss(fmap_r, fmap_g)
+                loss_gen, losses_gen = generator_loss(y_d_hat_g)
+                loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
+        optim_g.zero_grad()
+        scaler.scale(loss_gen_all).backward()
+        scaler.unscale_(optim_g)
+        grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
+        scaler.step(optim_g)
+        scaler.update()
+
+        if rank == 0:
+            if global_step % hps.train.log_interval == 0:
+                lr = optim_g.param_groups[0]['lr']
+                losses = [
+                    loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl
+                ]
+                logger.info('Train Epoch: {} [{:.0f}%]'.format(
+                    epoch, 100. * batch_idx / len(train_loader)))
+                logger.info([x.item() for x in losses] + [global_step, lr])
+
+                scalar_dict = {
+                    "loss/g/total": loss_gen_all,
+                    "loss/d/total": loss_disc_all,
+                    "learning_rate": lr,
+                    "grad_norm_d": grad_norm_d,
+                    "grad_norm_g": grad_norm_g
+                }
+                scalar_dict.update({
+                    "loss/g/fm": loss_fm,
+                    "loss/g/mel": loss_mel,
+                    "loss/g/dur": loss_dur,
+                    "loss/g/kl": loss_kl
+                })
+
+                scalar_dict.update({
+                    "loss/g/{}".format(i): v
+                    for i, v in enumerate(losses_gen)
+                })
+                scalar_dict.update({
+                    "loss/d_r/{}".format(i): v
+                    for i, v in enumerate(losses_disc_r)
+                })
+                scalar_dict.update({
+                    "loss/d_g/{}".format(i): v
+                    for i, v in enumerate(losses_disc_g)
+                })
+                image_dict = {
+                    "slice/mel_org":
+                    utils.plot_spectrogram_to_numpy(
+                        y_mel[0].data.cpu().numpy()),
+                    "slice/mel_gen":
+                    utils.plot_spectrogram_to_numpy(
+                        y_hat_mel[0].data.cpu().numpy()),
+                    "all/mel":
+                    utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
+                    "all/attn":
+                    utils.plot_alignment_to_numpy(attn[0,
+                                                       0].data.cpu().numpy())
+                }
+                utils.summarize(writer=writer,
+                                global_step=global_step,
+                                images=image_dict,
+                                scalars=scalar_dict)
+
+            if global_step % hps.train.eval_interval == 0:
+                evaluate(hps, net_g, eval_loader, writer_eval)
+                utils.save_checkpoint(
+                    net_g, optim_g, hps.train.learning_rate, epoch,
+                    os.path.join(hps.model_dir,
+                                 "G_{}.pth".format(global_step)))
+                utils.save_checkpoint(
+                    net_d, optim_d, hps.train.learning_rate, epoch,
+                    os.path.join(hps.model_dir,
+                                 "D_{}.pth".format(global_step)))
+        global_step += 1
+
+    if rank == 0:
+        logger.info('====> Epoch: {}'.format(epoch))
+
+
+def evaluate(hps, generator, eval_loader, writer_eval):
+    generator.eval()
+    with torch.no_grad():
+        for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths,
+                        speakers) in enumerate(eval_loader):
+            x, x_lengths = x.cuda(0), x_lengths.cuda(0)
+            spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0)
+            y, y_lengths = y.cuda(0), y_lengths.cuda(0)
+            speakers = speakers.cuda(0)
+
+            # remove else
+            x = x[:1]
+            x_lengths = x_lengths[:1]
+            spec = spec[:1]
+            spec_lengths = spec_lengths[:1]
+            y = y[:1]
+            y_lengths = y_lengths[:1]
+            speakers = speakers[:1]
+            break
+        y_hat, attn, mask, *_ = generator.module.infer(x,
+                                                       x_lengths,
+                                                       speakers,
+                                                       max_len=1000)
+        y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length
+
+        mel = spec_to_mel_torch(spec, hps.data.filter_length,
+                                hps.data.n_mel_channels,
+                                hps.data.sampling_rate, hps.data.mel_fmin,
+                                hps.data.mel_fmax)
+        y_hat_mel = mel_spectrogram_torch(
+            y_hat.squeeze(1).float(), hps.data.filter_length,
+            hps.data.n_mel_channels, hps.data.sampling_rate,
+            hps.data.hop_length, hps.data.win_length, hps.data.mel_fmin,
+            hps.data.mel_fmax)
+    image_dict = {
+        "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
+    }
+    audio_dict = {"gen/audio": y_hat[0, :, :y_hat_lengths[0]]}
+    if global_step == 0:
+        image_dict.update(
+            {"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
+        audio_dict.update({"gt/audio": y[0, :, :y_lengths[0]]})
+
+    utils.summarize(writer=writer_eval,
+                    global_step=global_step,
+                    images=image_dict,
+                    audios=audio_dict,
+                    audio_sampling_rate=hps.data.sampling_rate)
+    generator.train()
+
+
+if __name__ == "__main__":
+    main()

export/vits/transforms.py

@@ -0,0 +1,200 @@
+import numpy as np
+import torch
+from torch.nn import functional as F
+
+DEFAULT_MIN_BIN_WIDTH = 1e-3
+DEFAULT_MIN_BIN_HEIGHT = 1e-3
+DEFAULT_MIN_DERIVATIVE = 1e-3
+
+
+def piecewise_rational_quadratic_transform(
+        inputs,
+        unnormalized_widths,
+        unnormalized_heights,
+        unnormalized_derivatives,
+        inverse=False,
+        tails=None,
+        tail_bound=1.,
+        min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+        min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+        min_derivative=DEFAULT_MIN_DERIVATIVE):
+
+    if tails is None:
+        spline_fn = rational_quadratic_spline
+        spline_kwargs = {}
+    else:
+        spline_fn = unconstrained_rational_quadratic_spline
+        spline_kwargs = {'tails': tails, 'tail_bound': tail_bound}
+
+    outputs, logabsdet = spline_fn(
+        inputs=inputs,
+        unnormalized_widths=unnormalized_widths,
+        unnormalized_heights=unnormalized_heights,
+        unnormalized_derivatives=unnormalized_derivatives,
+        inverse=inverse,
+        min_bin_width=min_bin_width,
+        min_bin_height=min_bin_height,
+        min_derivative=min_derivative,
+        **spline_kwargs)
+    return outputs, logabsdet
+
+
+def searchsorted(bin_locations, inputs, eps=1e-6):
+    bin_locations[..., bin_locations.size(-1) - 1] += eps
+    return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
+
+
+def unconstrained_rational_quadratic_spline(
+        inputs,
+        unnormalized_widths,
+        unnormalized_heights,
+        unnormalized_derivatives,
+        inverse=False,
+        tails='linear',
+        tail_bound=1.,
+        min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+        min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+        min_derivative=DEFAULT_MIN_DERIVATIVE):
+    inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
+    outside_interval_mask = ~inside_interval_mask
+
+    outputs = torch.zeros_like(inputs)
+    logabsdet = torch.zeros_like(inputs)
+
+    if tails == 'linear':
+        unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
+        constant = np.log(np.exp(1 - min_derivative) - 1)
+        unnormalized_derivatives[..., 0] = constant
+        unnormalized_derivatives[..., unnormalized_derivatives.size(-1) - 1] = constant
+
+        outputs[outside_interval_mask] = inputs[outside_interval_mask]
+        logabsdet[outside_interval_mask] = 0
+    else:
+        raise RuntimeError('{} tails are not implemented.'.format(tails))
+
+    outputs[inside_interval_mask], logabsdet[
+        inside_interval_mask] = rational_quadratic_spline(
+            inputs=inputs[inside_interval_mask],
+            unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
+            unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
+            unnormalized_derivatives=unnormalized_derivatives[
+                inside_interval_mask, :],
+            inverse=inverse,
+            left=-tail_bound,
+            right=tail_bound,
+            bottom=-tail_bound,
+            top=tail_bound,
+            min_bin_width=min_bin_width,
+            min_bin_height=min_bin_height,
+            min_derivative=min_derivative)
+
+    return outputs, logabsdet
+
+
+def rational_quadratic_spline(inputs,
+                              unnormalized_widths,
+                              unnormalized_heights,
+                              unnormalized_derivatives,
+                              inverse=False,
+                              left=0.,
+                              right=1.,
+                              bottom=0.,
+                              top=1.,
+                              min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+                              min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+                              min_derivative=DEFAULT_MIN_DERIVATIVE):
+    if torch.min(inputs) < left or torch.max(inputs) > right:
+        raise ValueError('Input to a transform is not within its domain')
+
+    num_bins = unnormalized_widths.shape[-1]
+
+    if min_bin_width * num_bins > 1.0:
+        raise ValueError('Minimal bin width too large for the number of bins')
+    if min_bin_height * num_bins > 1.0:
+        raise ValueError('Minimal bin height too large for the number of bins')
+
+    widths = F.softmax(unnormalized_widths, dim=-1)
+    widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
+    cumwidths = torch.cumsum(widths, dim=-1)
+    cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0)
+    cumwidths = (right - left) * cumwidths + left
+    cumwidths[..., 0] = left
+    cumwidths[..., cumwidths.size(-1) - 1] = right
+    widths = cumwidths[..., 1:] - cumwidths[..., :-1]
+
+    derivatives = min_derivative + F.softplus(unnormalized_derivatives)
+
+    heights = F.softmax(unnormalized_heights, dim=-1)
+    heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
+    cumheights = torch.cumsum(heights, dim=-1)
+    cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0)
+    cumheights = (top - bottom) * cumheights + bottom
+    cumheights[..., 0] = bottom
+    cumheights[..., cumheights.size(-1) - 1] = top
+    heights = cumheights[..., 1:] - cumheights[..., :-1]
+
+    if inverse:
+        bin_idx = searchsorted(cumheights, inputs)[..., None]
+    else:
+        bin_idx = searchsorted(cumwidths, inputs)[..., None]
+
+    input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0]
+    input_bin_widths = widths.gather(-1, bin_idx)[..., 0]
+
+    input_cumheights = cumheights.gather(-1, bin_idx)[..., 0]
+    delta = heights / widths
+    input_delta = delta.gather(-1, bin_idx)[..., 0]
+
+    input_derivatives = derivatives.gather(-1, bin_idx)[..., 0]
+    input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[...,
+                                                                          0]
+
+    input_heights = heights.gather(-1, bin_idx)[..., 0]
+
+    if inverse:
+        a = (
+            ((inputs - input_cumheights) *
+             (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+             + input_heights * (input_delta - input_derivatives)))
+        b = (
+            input_heights * input_derivatives - (inputs - input_cumheights) *
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta))
+        c = -input_delta * (inputs - input_cumheights)
+
+        discriminant = b.pow(2) - 4 * a * c
+        assert (discriminant >= 0).all()
+
+        root = (2 * c) / (-b - torch.sqrt(discriminant))
+        outputs = root * input_bin_widths + input_cumwidths
+
+        theta_one_minus_theta = root * (1 - root)
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta)
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * root.pow(2) +
+            2 * input_delta * theta_one_minus_theta + input_derivatives *
+            (1 - root).pow(2))
+        logabsdet = torch.log(
+            derivative_numerator) - 2 * torch.log(denominator)
+
+        return outputs, -logabsdet
+    else:
+        theta = (inputs - input_cumwidths) / input_bin_widths
+        theta_one_minus_theta = theta * (1 - theta)
+
+        numerator = input_heights * (input_delta * theta.pow(2) +
+                                     input_derivatives * theta_one_minus_theta)
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta)
+        outputs = input_cumheights + numerator / denominator
+
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * theta.pow(2) +
+            2 * input_delta * theta_one_minus_theta + input_derivatives *
+            (1 - theta).pow(2))
+        logabsdet = torch.log(
+            derivative_numerator) - 2 * torch.log(denominator)
+
+        return outputs, logabsdet

export/vits/utils.py

@@ -0,0 +1,307 @@
+import argparse
+import glob
+import json
+import logging
+import os
+import subprocess
+import sys
+
+import numpy as np
+from scipy.io.wavfile import read
+import torch
+
+MATPLOTLIB_FLAG = False
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None):
+    assert os.path.isfile(checkpoint_path)
+    checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
+    iteration = checkpoint_dict['iteration']
+    learning_rate = checkpoint_dict['learning_rate']
+    if optimizer is not None:
+        optimizer.load_state_dict(checkpoint_dict['optimizer'])
+    saved_state_dict = checkpoint_dict['model']
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    new_state_dict = {}
+    for k, v in state_dict.items():
+        try:
+            new_state_dict[k] = saved_state_dict[k]
+        except Exception as e:
+            logger.info("%s is not in the checkpoint" % k)
+            new_state_dict[k] = v
+    if hasattr(model, 'module'):
+        model.module.load_state_dict(new_state_dict)
+    else:
+        model.load_state_dict(new_state_dict)
+    logger.info("Loaded checkpoint '{}' (iteration {})".format(
+        checkpoint_path, iteration))
+    return model, optimizer, learning_rate, iteration
+
+
+def save_checkpoint(model, optimizer, learning_rate, iteration,
+                    checkpoint_path):
+    logger.info(
+        "Saving model and optimizer state at iteration {} to {}".format(
+            iteration, checkpoint_path))
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    torch.save(
+        {
+            'model': state_dict,
+            'iteration': iteration,
+            'optimizer': optimizer.state_dict(),
+            'learning_rate': learning_rate
+        }, checkpoint_path)
+
+
+def summarize(
+        writer,
+        global_step,
+        scalars={},  # noqa
+        histograms={},  # noqa
+        images={},  # noqa
+        audios={},  # noqa
+        audio_sampling_rate=22050):
+    for k, v in scalars.items():
+        writer.add_scalar(k, v, global_step)
+    for k, v in histograms.items():
+        writer.add_histogram(k, v, global_step)
+    for k, v in images.items():
+        writer.add_image(k, v, global_step, dataformats='HWC')
+    for k, v in audios.items():
+        writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def latest_checkpoint_path(dir_path, regex="G_*.pth"):
+    f_list = glob.glob(os.path.join(dir_path, regex))
+    f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+    x = f_list[-1]
+    print(x)
+    return x
+
+
+def plot_spectrogram_to_numpy(spectrogram):
+    global MATPLOTLIB_FLAG
+    if not MATPLOTLIB_FLAG:
+        import matplotlib
+        matplotlib.use("Agg")
+        MATPLOTLIB_FLAG = True
+        mpl_logger = logging.getLogger('matplotlib')
+        mpl_logger.setLevel(logging.WARNING)
+    import matplotlib.pylab as plt
+    import numpy as np
+
+    fig, ax = plt.subplots(figsize=(10, 2))
+    im = ax.imshow(spectrogram,
+                   aspect="auto",
+                   origin="lower",
+                   interpolation='none')
+    plt.colorbar(im, ax=ax)
+    plt.xlabel("Frames")
+    plt.ylabel("Channels")
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
+    plt.close()
+    return data
+
+
+def plot_alignment_to_numpy(alignment, info=None):
+    global MATPLOTLIB_FLAG
+    if not MATPLOTLIB_FLAG:
+        import matplotlib
+        matplotlib.use("Agg")
+        MATPLOTLIB_FLAG = True
+        mpl_logger = logging.getLogger('matplotlib')
+        mpl_logger.setLevel(logging.WARNING)
+    import matplotlib.pylab as plt
+    import numpy as np
+
+    fig, ax = plt.subplots(figsize=(6, 4))
+    im = ax.imshow(alignment.transpose(),
+                   aspect='auto',
+                   origin='lower',
+                   interpolation='none')
+    fig.colorbar(im, ax=ax)
+    xlabel = 'Decoder timestep'
+    if info is not None:
+        xlabel += '\n\n' + info
+    plt.xlabel(xlabel)
+    plt.ylabel('Encoder timestep')
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
+    plt.close()
+    return data
+
+
+def load_wav_to_torch(full_path):
+    sampling_rate, data = read(full_path)
+    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_filepaths_and_text(filename, split="|"):
+    with open(filename, encoding='utf-8') as f:
+        filepaths_and_text = [line.strip().split(split) for line in f]
+    return filepaths_and_text
+
+
+def get_hparams(init=True):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c',
+                        '--config',
+                        type=str,
+                        default="./configs/base.json",
+                        help='JSON file for configuration')
+    parser.add_argument('-m',
+                        '--model',
+                        type=str,
+                        required=True,
+                        help='Model name')
+    parser.add_argument('--train_data',
+                        type=str,
+                        required=True,
+                        help='train data')
+    parser.add_argument('--val_data', type=str, required=True, help='val data')
+    parser.add_argument('--phone_table',
+                        type=str,
+                        required=True,
+                        help='phone table')
+    parser.add_argument('--speaker_table',
+                        type=str,
+                        default=None,
+                        help='speaker table, required for multiple speakers')
+
+    args = parser.parse_args()
+    model_dir = args.model
+
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+
+    config_path = args.config
+    config_save_path = os.path.join(model_dir, "config.json")
+    if init:
+        with open(config_path, "r", encoding='utf8') as f:
+            data = f.read()
+        with open(config_save_path, "w", encoding='utf8') as f:
+            f.write(data)
+    else:
+        with open(config_save_path, "r", encoding='utf8') as f:
+            data = f.read()
+    config = json.loads(data)
+    config['data']['training_files'] = args.train_data
+    config['data']['validation_files'] = args.val_data
+    config['data']['phone_table'] = args.phone_table
+    # 0 is kept for blank
+    config['data']['num_phones'] = len(open(args.phone_table).readlines()) + 1
+    if args.speaker_table is not None:
+        config['data']['speaker_table'] = args.speaker_table
+        # 0 is kept for unknown speaker
+        config['data']['n_speakers'] = len(
+            open(args.speaker_table).readlines()) + 1
+    else:
+        config['data']['n_speakers'] = 0
+
+    hparams = HParams(**config)
+    hparams.model_dir = model_dir
+    return hparams
+
+
+def get_hparams_from_dir(model_dir):
+    config_save_path = os.path.join(model_dir, "config.json")
+    with open(config_save_path, "r") as f:
+        data = f.read()
+    config = json.loads(data)
+
+    hparams = HParams(**config)
+    hparams.model_dir = model_dir
+    return hparams
+
+
+def get_hparams_from_file(config_path):
+    with open(config_path, "r") as f:
+        data = f.read()
+    config = json.loads(data)
+
+    hparams = HParams(**config)
+    return hparams
+
+
+def check_git_hash(model_dir):
+    source_dir = os.path.dirname(os.path.realpath(__file__))
+    if not os.path.exists(os.path.join(source_dir, ".git")):
+        logger.warn('''{} is not a git repository, therefore hash value
+                       comparison will be ignored.'''.format(source_dir))
+        return
+
+    cur_hash = subprocess.getoutput("git rev-parse HEAD")
+
+    path = os.path.join(model_dir, "githash")
+    if os.path.exists(path):
+        saved_hash = open(path).read()
+        if saved_hash != cur_hash:
+            logger.warn(
+                "git hash values are different. {}(saved) != {}(current)".
+                format(saved_hash[:8], cur_hash[:8]))
+    else:
+        open(path, "w").write(cur_hash)
+
+
+def get_logger(model_dir, filename="train.log"):
+    global logger
+    logger = logging.getLogger(os.path.basename(model_dir))
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    h = logging.FileHandler(os.path.join(model_dir, filename))
+    h.setLevel(logging.INFO)
+    h.setFormatter(formatter)
+    logger.addHandler(h)
+    return logger
+
+
+class HParams():
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if type(v) == dict:
+                v = HParams(**v)
+            self[k] = v
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __repr__(self):
+        return self.__dict__.__repr__()

pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "vits-onnx"
+version = "0.1.0"
+description = ""
+authors = ["ccds <ccdesue@163.com>"]
+readme = "README.md"
+packages = [{include = "vits_onnx"}]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+gradio = "^3.16.1"
+loguru = "^0.6.0"
+onnxruntime = "^1.13.1"
+pyopenjtalk = "^0.3.0"
+unidecode = "^1.3.6"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,77 @@
+aiohttp==3.8.3 ; python_version >= "3.9" and python_version < "4.0"
+aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "4.0"
+altair==4.2.0 ; python_version >= "3.9" and python_version < "4.0"
+anyio==3.6.2 ; python_version >= "3.9" and python_version < "4.0"
+async-timeout==4.0.2 ; python_version >= "3.9" and python_version < "4.0"
+attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==2.1.1 ; python_version >= "3.9" and python_version < "4.0"
+click==8.1.3 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows"
+coloredlogs==15.0.1 ; python_version >= "3.9" and python_version < "4.0"
+contourpy==1.0.6 ; python_version >= "3.9" and python_version < "4.0"
+cycler==0.11.0 ; python_version >= "3.9" and python_version < "4.0"
+cython==0.29.33 ; python_version >= "3.9" and python_version < "4.0"
+entrypoints==0.4 ; python_version >= "3.9" and python_version < "4.0"
+fastapi==0.89.0 ; python_version >= "3.9" and python_version < "4.0"
+ffmpy==0.3.0 ; python_version >= "3.9" and python_version < "4.0"
+flatbuffers==23.1.4 ; python_version >= "3.9" and python_version < "4.0"
+fonttools==4.38.0 ; python_version >= "3.9" and python_version < "4.0"
+frozenlist==1.3.3 ; python_version >= "3.9" and python_version < "4.0"
+fsspec==2022.11.0 ; python_version >= "3.9" and python_version < "4.0"
+gradio==3.16.1 ; python_version >= "3.9" and python_version < "4.0"
+h11==0.14.0 ; python_version >= "3.9" and python_version < "4.0"
+httpcore==0.16.3 ; python_version >= "3.9" and python_version < "4.0"
+httpx==0.23.3 ; python_version >= "3.9" and python_version < "4.0"
+humanfriendly==10.0 ; python_version >= "3.9" and python_version < "4.0"
+idna==3.4 ; python_version >= "3.9" and python_version < "4"
+jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
+jsonschema==4.17.3 ; python_version >= "3.9" and python_version < "4.0"
+kiwisolver==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
+linkify-it-py==1.0.3 ; python_version >= "3.9" and python_version < "4.0"
+loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0"
+markdown-it-py==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
+markdown-it-py[linkify,plugins]==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
+markupsafe==2.1.1 ; python_version >= "3.9" and python_version < "4.0"
+matplotlib==3.6.2 ; python_version >= "3.9" and python_version < "4.0"
+mdit-py-plugins==0.3.3 ; python_version >= "3.9" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
+mpmath==1.2.1 ; python_version >= "3.9" and python_version < "4.0"
+multidict==6.0.4 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.24.1 ; python_version < "4.0" and python_version >= "3.9"
+onnxruntime==1.13.1 ; python_version >= "3.9" and python_version < "4.0"
+orjson==3.8.4 ; python_version >= "3.9" and python_version < "4.0"
+packaging==23.0 ; python_version >= "3.9" and python_version < "4.0"
+pandas==1.5.2 ; python_version >= "3.9" and python_version < "4.0"
+pillow==9.4.0 ; python_version >= "3.9" and python_version < "4.0"
+protobuf==4.21.12 ; python_version >= "3.9" and python_version < "4.0"
+pycryptodome==3.16.0 ; python_version >= "3.9" and python_version < "4.0"
+pydantic==1.10.4 ; python_version >= "3.9" and python_version < "4.0"
+pydub==0.25.1 ; python_version >= "3.9" and python_version < "4.0"
+pyopenjtalk==0.3.0 ; python_version >= "3.9" and python_version < "4.0"
+pyparsing==3.0.9 ; python_version >= "3.9" and python_version < "4.0"
+pyreadline3==3.4.1 ; sys_platform == "win32" and python_version >= "3.9" and python_version < "4.0"
+pyrsistent==0.19.3 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+python-multipart==0.0.5 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2022.7 ; python_version >= "3.9" and python_version < "4.0"
+pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.28.1 ; python_version >= "3.9" and python_version < "4"
+rfc3986[idna2008]==1.5.0 ; python_version >= "3.9" and python_version < "4.0"
+setuptools-scm==7.1.0 ; python_version >= "3.9" and python_version < "4.0"
+setuptools==65.6.3 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+sniffio==1.3.0 ; python_version >= "3.9" and python_version < "4.0"
+starlette==0.22.0 ; python_version >= "3.9" and python_version < "4.0"
+sympy==1.11.1 ; python_version >= "3.9" and python_version < "4.0"
+tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
+toolz==0.12.0 ; python_version >= "3.9" and python_version < "4.0"
+tqdm==4.64.1 ; python_version >= "3.9" and python_version < "4.0"
+typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0"
+uc-micro-py==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
+unidecode==1.3.6 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.13 ; python_version >= "3.9" and python_version < "4"
+uvicorn==0.20.0 ; python_version >= "3.9" and python_version < "4.0"
+websockets==10.4 ; python_version >= "3.9" and python_version < "4.0"
+win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32"
+yarl==1.8.2 ; python_version >= "3.9" and python_version < "4.0"

setup.sh

@@ -0,0 +1,22 @@
+# sudo su 
+conda create -n dl  python=3.9 -y
+conda init bash
+bash
+
+conda activate dl
+export POETRY_VERSION=1.3.1
+export DEBIAN_FRONTEND=noninteractive && \
+    sudo apt-get update && \
+    sudo apt-get install cmake build-essential  -y --no-install-recommends && \
+    pip install poetry==$POETRY_VERSION 
+
+
+poetry export -f requirements.txt -o requirements.txt --without dev --without test  --without-hashes  && \
+pip install --upgrade pip  && \
+pip install -r requirements.txt  
+
+
+
+function run{
+    python -m app.main
+}

util/build_docker.sh

@@ -0,0 +1,2 @@
+export DOCKER_BUILDKIT=1
+docker build -f Dockerfile -t ccdesue/vits_demo .

util/extract_w.py

@@ -0,0 +1,20 @@
+# from tabnanny import check
+import torch
+import pathlib
+
+path = r"/workspaces/vits_web_demo/export/model/D_second.pth"
+model_path = pathlib.Path(path)
+
+assert model_path.exists(), "model path does not exist"
+
+checkpoint = torch.load(str(model_path), map_location='cpu')
+
+state_file = checkpoint['model']
+iteration = checkpoint['iteration']
+
+out_path = model_path.parent / pathlib.Path("19_"+str(iteration)+'_demo'+'.pth')
+out_path = str(out_path)
+torch.save({'model': state_file,
+            'iteration': iteration,
+            'optimizer': None,
+            'learning_rate': None}, out_path)