Repository: NatLibFi/Annif (language: python)

Issue:
Simplemma analyzer
Just like with spaCy (see #374) we could add an analyzer that uses [simplemma](https://github.com/adbar/simplemma/) for lemmatization. This is a very fast and lightweight multilingual lemmatizer which currently supports 38 languages. The lemmatization accuracy may not be as high as with e.g. Stanza (see #539) but in practice that doesn't seem to matter much based on experiments I've performed.

Simplemma is implemented in pure Python without external dependencies so I think it should be possible to include this as a core feature, not an optional one, unless there are any unexpected problems with e.g. supported Python versions.
Simplemma analyzer
Just like with spaCy (see #374) we could add an analyzer that uses [simplemma](https://github.com/adbar/simplemma/) for lemmatization. This is a very fast and lightweight multilingual lemmatizer which currently supports 38 languages. The lemmatization accuracy may not be as high as with e.g. Stanza (see #539) but in practice that doesn't seem to matter much based on experiments I've performed.

Simplemma is implemented in pure Python without external dependencies so I think it should be possible to include this as a core feature, not an optional one, unless there are any unexpected problems with e.g. supported Python versions.


Discussion / hints:

Fix patch:
diff --git a/annif/analyzer/__init__.py b/annif/analyzer/__init__.py
index c3427d953..954c315f6 100644
--- a/annif/analyzer/__init__.py
+++ b/annif/analyzer/__init__.py
@@ -3,6 +3,7 @@
 import re
 from . import simple
 from . import snowball
+from . import simplemma
 import annif
 from annif.util import parse_args
 
@@ -30,6 +31,7 @@ def get_analyzer(analyzerspec):
 
 register_analyzer(simple.SimpleAnalyzer)
 register_analyzer(snowball.SnowballAnalyzer)
+register_analyzer(simplemma.SimplemmaAnalyzer)
 
 # Optional analyzers
 try:
diff --git a/annif/analyzer/simplemma.py b/annif/analyzer/simplemma.py
new file mode 100644
index 000000000..71df0d295
--- /dev/null
+++ b/annif/analyzer/simplemma.py
@@ -0,0 +1,26 @@
+"""Simplemma analyzer for Annif, based on simplemma lemmatizer."""
+
+import functools
+import simplemma
+from . import analyzer
+
+
+class SimplemmaAnalyzer(analyzer.Analyzer):
+    name = "simplemma"
+
+    def __init__(self, param, **kwargs):
+        self.lang = param
+        self.langdata = None
+        super().__init__(**kwargs)
+
+    def __getstate__(self):
+        """Return the state of the object for pickling purposes. The langdata
+        field is set to None as it's more efficient to use load_data."""
+
+        return {'lang': self.lang, 'langdata': None}
+
+    @functools.lru_cache(maxsize=500000)
+    def _normalize_word(self, word):
+        if self.langdata is None:
+            self.langdata = simplemma.load_data(self.lang)
+        return simplemma.lemmatize(word, self.langdata)
diff --git a/setup.py b/setup.py
index d17b08c29..a9e050b5a 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,8 @@ def read(fname):
         'optuna==2.10.*',
         'stwfsapy==0.3.*',
         'python-dateutil',
-        'tomli==2.0.*'
+        'tomli==2.0.*',
+        'simplemma==0.6.*'
     ],
     tests_require=['py', 'pytest', 'requests'],
     extras_require={


Test patch:
diff --git a/tests/test_analyzer_simplemma.py b/tests/test_analyzer_simplemma.py
new file mode 100644
index 000000000..c3fc36d76
--- /dev/null
+++ b/tests/test_analyzer_simplemma.py
@@ -0,0 +1,19 @@
+"""Unit tests for simplemma analyzer in Annif"""
+
+import pytest
+import annif.analyzer
+
+simplemma = pytest.importorskip("annif.analyzer.simplemma")
+
+
+def test_simplemma_getstate():
+    analyzer = annif.analyzer.get_analyzer("simplemma(fi)")
+    state = analyzer.__getstate__()
+    assert state == {'lang': 'fi', 'langdata': None}
+
+
+def test_simplemma_finnish_analyzer_normalize_word():
+    analyzer = annif.analyzer.get_analyzer("simplemma(fi)")
+    assert analyzer._normalize_word("xyzzy") == "xyzzy"
+    assert analyzer._normalize_word("vanhat") == "vanha"
+    assert analyzer._normalize_word("koirien") == "koira"