Repository: r9y9/pyopenjtalk (language: python)

Issue:
Embed C++ source into the repository
With this, users will not need to build open_jtalk C++ library in advance. 

This can be done as in https://github.com/r9y9/pysptk. Compiling C++ sources with cython would be tricky though, it should be possible.


Discussion / hints:

Fix patch:
diff --git a/.github/stale.yml b/.github/stale.yml
deleted file mode 100644
index 4553ebb..0000000
--- a/.github/stale.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 60
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
-  - roadmap
-  - bug
-  - design
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: true
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: true
-
-# Label to use when marking as stale
-staleLabel: wontfix
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-only: issues
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 0000000..1641a12
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install numpy scipy cython
+        pip install -e .
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        flake8
+    - name: Test with pytest
+      run: |
+        pip install pytest
+        pytest
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..369c4e7
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "lib/open_jtalk"]
+	path = lib/open_jtalk
+	url = https://github.com/r9y9/open_jtalk.git
diff --git a/.travis.yml b/.travis.yml
index fe4c3e9..ec0ed4a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 language: python
 
 python:
-  - "3.6"
   - "3.7"
   - "3.8"
 
@@ -10,22 +9,8 @@ notifications:
 
 os:
   - linux
-#  - osx
 
 before_install:
-  - export LD_LIBRARY_PATH=/usr/local/lib
-  # hts_engine_API
-  - cd $HOME
-  - git clone https://github.com/r9y9/hts_engine_API.git && cd hts_engine_API/src
-  - sudo ./waf configure build
-  - sudo ./waf install
-  # open_jtalk
-  - cd $HOME
-  - git clone https://github.com/r9y9/open_jtalk.git && cd open_jtalk/src
-  - mkdir -p build && cd build
-  - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON ..
-  - make -j
-  - sudo make install
   # anaconda
   - if [["$TRAVIS_PYTHON_VERSION" == "2.7"]]; then
       wget http://repo.continuum.io/miniconda/Miniconda-3.8.3-Linux-x86_64.sh -O miniconda.sh;
diff --git a/README.md b/README.md
index 2e4c1e4..0a07bcf 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # pyopenjtalk
 
 [![PyPI](https://img.shields.io/pypi/v/pyopenjtalk.svg)](https://pypi.python.org/pypi/pyopenjtalk)
+[![Python package](https://github.com/r9y9/pyopenjtalk/actions/workflows/ci.yaml/badge.svg)](https://github.com/r9y9/pyopenjtalk/actions/workflows/ci.yaml)
 [![Build Status](https://travis-ci.org/r9y9/pyopenjtalk.svg?branch=master)](https://travis-ci.org/r9y9/pyopenjtalk)
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE.md)
 [![DOI](https://zenodo.org/badge/143748865.svg)](https://zenodo.org/badge/latestdoi/143748865)
diff --git a/lib/open_jtalk b/lib/open_jtalk
new file mode 160000
index 0000000..2221314
--- /dev/null
+++ b/lib/open_jtalk
@@ -0,0 +1,1 @@
+Subproject commit 22213146d3491d708ec9b827851de8c0bbca29d7
diff --git a/pyopenjtalk/legacy.py b/pyopenjtalk/legacy.py
deleted file mode 100644
index 071be91..0000000
--- a/pyopenjtalk/legacy.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# coding: utf-8
-
-from __future__ import division, print_function, absolute_import
-
-import pkg_resources
-import tempfile
-import subprocess
-from os.path import join, dirname
-import os
-
-htsvoice_path = pkg_resources.resource_filename(__name__, "htsvoice/mei_normal.htsvoice")
-dict_path = "/usr/local/dic"
-
-
-def openjtalk(text, cleanup=True, verbose=0):
-    with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".txt", delete=False) as f:
-        f.write(text)
-        text_path = f.name
-
-    with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".wav") as f:
-        wav_path = f.name
-
-    with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".trace") as f:
-        trace_path = f.name
-
-    cmd = ["/usr/local/bin/open_jtalk", "-m", htsvoice_path, "-x", dict_path,
-           text_path, "-ow", wav_path, "-ot", trace_path]
-
-    # Run
-    subprocess.check_call(cmd)
-
-    prons = []
-    labels = []
-    pron_in = False
-    label_in = False
-    global_in = False
-    global_params = {"Sampring frequency": None, "Frame period": None}
-    with open(trace_path) as f:
-        for l in f:
-            l = l[:-1]
-            if verbose > 0 and (pron_in or label_in):
-                print(l)
-            if l.startswith("[Text analysis result]"):
-                pron_in = True
-                continue
-            if pron_in and len(l) == 0:
-                pron_in = False
-            if pron_in:
-                row = l.split(",")
-                hinsi = row[1]
-                if hinsi == "記号":
-                    prons.append(row[0])
-                else:
-                    prons.append(row[9])
-
-            if l.startswith("[Output label]"):
-                label_in = True
-                continue
-            if label_in and len(l) == 0:
-                label_in = False
-            if label_in:
-                labels.append(l)
-
-            if l.startswith("[Global parameter]"):
-                global_in = True
-                continue
-            if global_in:
-                row = list(map(lambda s: s.strip(), l.split("->")))
-                if len(row) < 2:
-                    continue
-                k = row[0]
-                if k in global_params.keys():
-                    v = row[1]
-                    for s in ["(Hz)", "(point)"]:
-                        v = v.replace(s, "")
-                    global_params[k] = v
-
-    if cleanup:
-        for p in [text_path, wav_path, trace_path]:
-            os.remove(p)
-
-    # Fix typ..
-    for b, a in [("Sampring frequency", "Sampling frequency")]:
-        assert b in global_params
-        val = global_params[b]
-        del global_params[b]
-        global_params[a] = val
-
-    return prons, labels, global_params
diff --git a/pyopenjtalk/openjtalk/jpcommon.pxd b/pyopenjtalk/openjtalk/jpcommon.pxd
index 58d4a6a..8e86bea 100644
--- a/pyopenjtalk/openjtalk/jpcommon.pxd
+++ b/pyopenjtalk/openjtalk/jpcommon.pxd
@@ -2,7 +2,7 @@
 
 from libc.stdio cimport FILE
 
-cdef extern from "openjtalk/jpcommon.h":
+cdef extern from "jpcommon.h":
     cdef cppclass JPCommonNode:
        char *pron
        char *pos
diff --git a/pyopenjtalk/openjtalk/mecab.pxd b/pyopenjtalk/openjtalk/mecab.pxd
index 7ce118e..bd367c7 100644
--- a/pyopenjtalk/openjtalk/mecab.pxd
+++ b/pyopenjtalk/openjtalk/mecab.pxd
@@ -1,6 +1,6 @@
 # distutils: language = c++
 
-cdef extern from "openjtalk/mecab.h":
+cdef extern from "mecab.h":
     cdef cppclass Mecab:
         char **feature
         int size
diff --git a/pyopenjtalk/openjtalk/mecab2njd.pxd b/pyopenjtalk/openjtalk/mecab2njd.pxd
index 327817e..be57ccc 100644
--- a/pyopenjtalk/openjtalk/mecab2njd.pxd
+++ b/pyopenjtalk/openjtalk/mecab2njd.pxd
@@ -2,5 +2,5 @@
 
 from .njd cimport NJD
 
-cdef extern from "openjtalk/mecab2njd.h":
+cdef extern from "mecab2njd.h":
     void mecab2njd(NJD * njd, char **feature, int size);
diff --git a/pyopenjtalk/openjtalk/njd.pxd b/pyopenjtalk/openjtalk/njd.pxd
index f40ead2..38d3887 100644
--- a/pyopenjtalk/openjtalk/njd.pxd
+++ b/pyopenjtalk/openjtalk/njd.pxd
@@ -2,7 +2,7 @@
 
 from libc.stdio cimport FILE
 
-cdef extern from "openjtalk/njd.h":
+cdef extern from "njd.h":
     cdef cppclass NJDNode:
         char *string
         char *pos
@@ -78,20 +78,20 @@ cdef extern from "openjtalk/njd.h":
     void NJD_refresh(NJD * njd)
     void NJD_clear(NJD * wl)
 
-cdef extern from "openjtalk/njd_set_accent_phrase.h":
+cdef extern from "njd_set_accent_phrase.h":
     void njd_set_accent_phrase(NJD * njd)
 
-cdef extern from "openjtalk/njd_set_accent_type.h":
+cdef extern from "njd_set_accent_type.h":
     void njd_set_accent_type(NJD * njd)
 
-cdef extern from "openjtalk/njd_set_digit.h":
+cdef extern from "njd_set_digit.h":
     void njd_set_digit(NJD * njd)
 
-cdef extern from "openjtalk/njd_set_long_vowel.h":
+cdef extern from "njd_set_long_vowel.h":
     void njd_set_long_vowel(NJD * njd)
 
-cdef extern from "openjtalk/njd_set_pronunciation.h":
+cdef extern from "njd_set_pronunciation.h":
     void njd_set_pronunciation(NJD * njd)
 
-cdef extern from "openjtalk/njd_set_unvoiced_vowel.h":
+cdef extern from "njd_set_unvoiced_vowel.h":
     void njd_set_unvoiced_vowel(NJD * njd)
diff --git a/pyopenjtalk/openjtalk/njd2jpcommon.pxd b/pyopenjtalk/openjtalk/njd2jpcommon.pxd
index c376287..8309288 100644
--- a/pyopenjtalk/openjtalk/njd2jpcommon.pxd
+++ b/pyopenjtalk/openjtalk/njd2jpcommon.pxd
@@ -3,5 +3,5 @@
 from .jpcommon cimport JPCommon
 from .njd cimport NJD
 
-cdef extern from "openjtalk/njd2jpcommon.h":
+cdef extern from "njd2jpcommon.h":
     void njd2jpcommon(JPCommon * jpcommon, NJD * njd)
diff --git a/pyopenjtalk/openjtalk/text2mecab.pxd b/pyopenjtalk/openjtalk/text2mecab.pxd
index d46b0c1..6081757 100644
--- a/pyopenjtalk/openjtalk/text2mecab.pxd
+++ b/pyopenjtalk/openjtalk/text2mecab.pxd
@@ -1,4 +1,4 @@
 # distutils: language = c++
 
-cdef extern from "openjtalk/text2mecab.h":
+cdef extern from "text2mecab.h":
     void text2mecab(char *output, const char *input)
diff --git a/setup.py b/setup.py
index 89e646f..bf8e539 100644
--- a/setup.py
+++ b/setup.py
@@ -10,21 +10,11 @@
 import numpy as np
 import os
 from glob import glob
-from os.path import join
+from os.path import join, exists
+from subprocess import run
 
 version = '0.0.3'
 
-openjtalk_install_prefix = os.environ.get(
-    "OPEN_JTALK_INSTALL_PREFIX", "/usr/local/")
-
-openjtalk_include_top = join(openjtalk_install_prefix, "include")
-openjtalk_library_path = join(openjtalk_install_prefix, "lib")
-
-lib_candidates = list(filter(lambda l: l.startswith("libopenjtalk."),
-                             os.listdir(join(openjtalk_library_path))))
-if len(lib_candidates) == 0:
-    raise OSError("openjtalk library cannot be found")
-
 min_cython_ver = '0.21.0'
 try:
     import Cython
@@ -51,20 +41,49 @@
     if not os.path.exists(join("pyopenjtalk", "openjtalk" + ext)):
         raise RuntimeError("Cython is required to generate C++ code")
 
-ext_modules = cythonize(
-    [Extension(
-        name="pyopenjtalk.openjtalk",
-        sources=[
-            join("pyopenjtalk", "openjtalk" + ext),
-        ],
-        include_dirs=[np.get_include(),
-                      join(openjtalk_include_top)],
-        library_dirs=[openjtalk_library_path],
-        libraries=["openjtalk"],
-        extra_compile_args=[],
-        extra_link_args=[],
-        language="c++")],
-)
+# open_jtalk sources
+src_top = join("lib", "open_jtalk", "src")
+
+# generate config.h for mecab
+# NOTE: need to run cmake to generate config.h
+# we could do it on python side but it would be very tricky,
+# so far let's use cmake tool
+if not exists(join(src_top, "mecab", "src", "config.h")):
+    cwd = os.getcwd()
+    build_dir = join(src_top, "build")
+    os.makedirs(build_dir, exist_ok=True)
+    os.chdir(build_dir)
+    run(["cmake", ".."])
+    os.chdir(cwd)
+
+all_src = []
+include_dirs = []
+for s in [
+    "jpcommon", "mecab/src", "mecab2njd", "njd", "njd2jpcommon",
+    "njd_set_accent_phrase", "njd_set_accent_type",
+    "njd_set_digit", "njd_set_long_vowel", "njd_set_pronunciation",
+    "njd_set_unvoiced_vowel", "text2mecab",
+]:
+    all_src += glob(join(src_top, s, "*.c"))
+    all_src += glob(join(src_top, s, "*.cpp"))
+    include_dirs.append(join(os.getcwd(), src_top, s))
+
+# define core cython module
+ext_modules = [Extension(
+    name="pyopenjtalk.openjtalk",
+    sources=[join("pyopenjtalk", "openjtalk" + ext)] + all_src,
+    include_dirs=[np.get_include()] + include_dirs,
+    extra_compile_args=[],
+    extra_link_args=[],
+    language="c++",
+    define_macros=[
+        ("HAVE_CONFIG_H", None),
+        ("DIC_VERSION", 102), ("MECAB_DEFAULT_RC", "\"dummy\""),
+        ("PACKAGE", "\"open_jtalk\""),
+        ("VERSION", "\"1.10\""),
+        ("CHARSET_UTF_8", None),
+    ]
+)]
 
 # Adapted from https://github.com/pytorch/pytorch
 cwd = os.path.dirname(os.path.abspath(__file__))


Test patch:
diff --git a/tests/test_legacy.py b/tests/test_legacy.py
deleted file mode 100644
index f71f1df..0000000
--- a/tests/test_legacy.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from pyopenjtalk.legacy import openjtalk
-from nose.plugins.attrib import attr
-
-
-@attr("local_only")
-def test_legacy():
-    prons, labels, params = openjtalk("こんにちは")
-    for l in labels:
-        print(l)
-    assert "".join(prons) == "コンニチワ"