refactor: change translation method

.gitignore

@@ -0,0 +1,207 @@
+### venv template
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+
+### PythonVanilla template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+
+### VirtualEnv template
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+
+### Python template
+# Byte-compiled / optimized / DLL files
+
+# C extensions
+
+# Distribution / packaging
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+
+# Unit test / coverage reports
+
+# Translations
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+/flagged/

.idea/frasaria.iml

@@ -2,9 +2,10 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.9 (frasaria) (2)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (frasaria-spaces)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,7 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.10 (frasaria-spaces)" />
+  </component>
   <component name="MarkdownSettingsMigration">
     <option name="stateVersion" value="1" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (frasaria) (2)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (frasaria-spaces)" project-jdk-type="Python SDK" />
 </project>

app.py

@@ -1,9 +1,11 @@
 import gradio as gr
 from frasaria.frasaria import paraphrase_text
 
-gr.Interface(
-    paraphrase_text,
-    inputs=[gr.inputs.Textbox(lines=5, placeholder=None, label='Text'),
-            gr.Radio(['id', 'en'], value='id', label='Source Language')],
-    outputs=[gr.outputs.Textbox(label=None)]
-).launch()
+demo = gr.Interface(
+    fn=paraphrase_text,
+    inputs=gr.Textbox(lines=5, placeholder=None, label='Text'),
+    outputs=gr.Textbox(label=None)
+)
+
+if __name__ == '__main__':
+    demo.launch()

frasaria/frasaria.py

@@ -1,26 +1,17 @@
-import gradio as gr
-from frasaria import parrot, translator_id_to_en, translator_en_to_id
+from frasaria import parrot, translator
 import warnings
+
 warnings.filterwarnings('ignore')
 
+
 def split_sentences(text: str):
-    return [sentence.strip() for sentence in text.split('.')]
+    return [sentence.strip() for sentence in text.split(".")]
 
-def translate(text, source, target='en'):
-    if source == 'en':
-        return translator_en_to_id(text)
-    else:
-        return translator_id_to_en(text)
 
-def paraphrase(parrot, phrase):
-    para_phrases = parrot.augment(phrase, max_return_phrases=10, do_diverse=True)
+def paraphrase(phrases):
+    para_phrases = parrot.augment(phrases, max_return_phrases=10, do_diverse=True)
     max_score = 0
-    # sometimes the paraphrasing doesn't return anything, if so, return the original
-    try:
-        paraphrased = para_phrases[0][0]
-    except TypeError:
-        return phrase
-
+    paraphrased = para_phrases[0][0]
     for para_phrase in para_phrases:
         score = para_phrase[-1]
         if score > max_score:
@@ -28,21 +19,22 @@ def paraphrase(parrot, phrase):
             paraphrased = para_phrase[0]
     return paraphrased
 
-def paraphrase_text(text, source_lang):
-    phrases = split_sentences(text)
-    print('split_original phrases', phrases)
 
-    if source_lang == 'id':
-        en_sentences = []
-        for phrase in phrases:
-            en_sentences.append(translate(phrase, source_lang)[0]['translation_text'])
-        phrases = en_sentences
+def paraphrase_text(phrase):
+    phrase = phrase.replace('"', "'")
+    result = translator.translate_text(phrase, target_lang="EN-US")
 
-    print('phrase after translated or not', phrases)
+    # paraphrase
     sentences = []
-    for phrase in phrases:
-        sentences.append(paraphrase(parrot, phrase))
-
-    paraphrased = '. '.join(sentences)
-    source_lang_paraphrased = translate(paraphrased, 'en', 'id')
-    return source_lang_paraphrased[0]['translation_text']
+    for translated in split_sentences(result.text)[:-1]:
+        sentences.append(paraphrase(translated))
+
+    text = '. '.join(list(map(lambda x: x.capitalize(), sentences))) + '.'
+    if result.detected_source_lang == 'EN':
+        return text
+
+    # translate back to whatever the source language is
+    src_lang_text = translator.translate_text(
+        text, target_lang=result.detected_source_lang
+    )
+    return src_lang_text.text

requirements.txt

@@ -1,3 +1,7 @@
-transformers
-torch
-git+https://github.com/PrithivirajDamodaran/Parrot_Paraphraser.git
+deepl==1.17.0
+parrot @ git+https://github.com/PrithivirajDamodaran/Parrot_Paraphraser.git
+torch==2.3.0
+gradio==4.28.3
+huggingface-hub==0.22.2
+python-dotenv==1.0.1
+protobuf==5.26.1