Repository: adbar/simplemma

Issue:
Separate scripts used to create the dictionaries from actual src & tests
`tests/udscore.py` and `src/dictionary_pickler.py` seem to be scripts used to create the dictionaries and validate their quality.

I suggest that they are moved to a separate folder `training` or whatever you want

Comments:

Patch:
diff --git a/simplemma/__init__.py b/simplemma/__init__.py
index 74510e6..6a553e8 100644
--- a/simplemma/__init__.py
+++ b/simplemma/__init__.py
@@ -7,6 +7,7 @@ __license__ = "MIT"
 __version__ = "0.9.1"
 
 
+from .dictionary_factory import DictionaryFactory, DefaultDictionaryFactory
 from .language_detector import LanguageDetector, in_target_language, langdetect
 from .lemmatizer import Lemmatizer, lemmatize, lemma_iterator, text_lemmatizer, is_known
 from .tokenizer import Tokenizer, RegexTokenizer, simple_tokenizer
@@ -16,5 +17,3 @@ from .token_sampler import (
     MostCommonTokenSampler,
     RelaxedMostCommonTokenSampler,
 )
-from .dictionary_factory import DictionaryFactory, DefaultDictionaryFactory
-from .dictionary_pickler import *
diff --git a/simplemma/dictionary_pickler.py b/training/dictionary_pickler.py
similarity index 90%
rename from simplemma/dictionary_pickler.py
rename to training/dictionary_pickler.py
index 5f400ee..f0b92e0 100644
--- a/simplemma/dictionary_pickler.py
+++ b/training/dictionary_pickler.py
@@ -7,15 +7,11 @@ from operator import itemgetter
 from pathlib import Path
 from typing import Dict, List, Optional
 
-try:
-    from .dictionary_factory import SUPPORTED_LANGUAGES
-    from .strategies.defaultrules import DEFAULT_RULES
-    from .utils import levenshtein_dist
-# local error, also ModuleNotFoundError for Python >= 3.6
-except ImportError:  # pragma: no cover
-    from dictionary_factory import SUPPORTED_LANGUAGES  # type: ignore
-    from strategies.defaultrules import DEFAULT_RULES  # type: ignore
-    from utils import levenshtein_dist  # type: ignore
+import simplemma
+from simplemma.dictionary_factory import SUPPORTED_LANGUAGES
+from simplemma.strategies.defaultrules import DEFAULT_RULES
+from simplemma.utils import levenshtein_dist
+
 
 LOGGER = logging.getLogger(__name__)
 
@@ -138,7 +134,7 @@ def _pickle_dict(
         mydict = dict(sorted(mydict.items(), key=itemgetter(1)))
     if filepath is None:
         filename = f"data/{langcode}.plzma"
-        filepath = str(Path(__file__).parent / filename)
+        filepath = str(Path(simplemma.__file__).parent / filename)
     with lzma.open(filepath, "wb") as filehandle:  # , filters=my_filters, preset=9
         pickle.dump(mydict, filehandle, protocol=4)
     LOGGER.debug("%s %s", langcode, len(mydict))
diff --git a/eval/eval-requirements.txt b/training/eval/eval-requirements.txt
similarity index 100%
rename from eval/eval-requirements.txt
rename to training/eval/eval-requirements.txt
diff --git a/eval/udscore.py b/training/eval/udscore.py
similarity index 84%
rename from eval/udscore.py
rename to training/eval/udscore.py
index 34e3a9c..e6bd507 100644
--- a/eval/udscore.py
+++ b/training/eval/udscore.py
@@ -5,7 +5,8 @@ from collections import Counter
 from os import makedirs, path
 
 from conllu import parse_incr  # type: ignore
-from simplemma import lemmatize
+from simplemma import Lemmatizer, DefaultDictionaryFactory
+from simplemma.strategies.default import DefaultStrategy
 
 if not path.exists("csv"):
     makedirs("csv")
@@ -66,6 +67,16 @@ for filedata in data_files:
     with open(filename, "r", encoding="utf-8") as myfile:
         data_file = myfile.read()
     start = time.time()
+    dictionary_factory = DefaultDictionaryFactory()
+    strategies = DefaultStrategy(greedy=False)
+    lemmatizer = Lemmatizer(
+        dictionary_factory=dictionary_factory,
+        lemmatization_strategy=DefaultStrategy(greedy=False),
+    )
+    greedy_lemmatizer = Lemmatizer(
+        dictionary_factory=dictionary_factory,
+        lemmatization_strategy=DefaultStrategy(greedy=True),
+    )
     print("==", filedata, "==")
     for tokenlist in parse_incr(data_file):
         for token in tokenlist:
@@ -75,13 +86,10 @@ for filedata in data_files:
                 continue
 
             initial = bool(token["id"] == 1)
+            token_form = token["form"].lower() if initial else token["form"]
 
-            greedy_candidate = lemmatize(
-                token["form"], lang=language, greedy=True, initial=initial
-            )
-            candidate = lemmatize(
-                token["form"], lang=language, greedy=False, initial=initial
-            )
+            candidate = lemmatizer.lemmatize(token_form, lang=language)
+            greedy_candidate = greedy_lemmatizer.lemmatize(token_form, lang=language)
 
             if token["upos"] in ("ADJ", "NOUN"):
                 focus_total += 1


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4ebd08b..c73b552 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -86,11 +86,11 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
     - name: Code format with black
-      run: black --check --diff simplemma tests
+      run: black --check --diff simplemma training tests
 
     - name: Type checking with mypy
       if: matrix.python-version != '3.6'
-      run: mypy -p simplemma -p tests
+      run: mypy -p simplemma -p training -p tests
 
     - name: Test with pytest
       run: pytest --cov=./ --cov-report=xml
diff --git a/tests/test_dictionary_pickler.py b/tests/test_dictionary_pickler.py
index 1dad1fa..95dd1bb 100644
--- a/tests/test_dictionary_pickler.py
+++ b/tests/test_dictionary_pickler.py
@@ -1,7 +1,7 @@
 import os
 import tempfile
 
-from simplemma import dictionary_pickler
+from training import dictionary_pickler
 
 TEST_DIR = os.path.abspath(os.path.dirname(__file__))