Update spacy pipeline to 3.2.2

README.md

@@ -14,13 +14,13 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.9078212291
+      value: 0.9094751673
     - name: NER Recall
       type: recall
-      value: 0.9142053446
+      value: 0.9078762307
     - name: NER F Score
       type: f_score
-      value: 0.9110021023
+      value: 0.9086749956
   - task:
       name: TAG
       type: token-classification
@@ -48,7 +48,7 @@ model-index:
     metrics:
     - name: Lemma Accuracy
       type: accuracy
-      value: 0.9863170988
+      value: 0.9864127835
   - task:
       name: UNLABELED_DEPENDENCIES
       type: token-classification
@@ -76,10 +76,10 @@ Hungarian transformer pipeline (huBert) for HuSpaCy. Components: transformer, se
 | Feature | Description |
 | --- | --- |
 | **Name** | `hu_core_news_trf` |
-| **Version** | `3.2.1` |
+| **Version** | `3.2.2` |
 | **spaCy** | `>=3.2.4,<3.3.0` |
-| **Default Pipeline** | `transformer`, `senter`, `tagger`, `morphologizer`, `experimental_edit_tree_lemmatizer`, `experimental_arc_predicter`, `experimental_arc_labeler`, `ner` |
-| **Components** | `transformer`, `senter`, `tagger`, `morphologizer`, `experimental_edit_tree_lemmatizer`, `experimental_arc_predicter`, `experimental_arc_labeler`, `ner` |
+| **Default Pipeline** | `transformer`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `experimental_edit_tree_lemmatizer`, `lemma_smoother`, `experimental_arc_predicter`, `experimental_arc_labeler`, `ner` |
+| **Components** | `transformer`, `senter`, `tagger`, `morphologizer`, `lookup_lemmatizer`, `experimental_edit_tree_lemmatizer`, `lemma_smoother`, `experimental_arc_predicter`, `experimental_arc_labeler`, `ner` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | [UD Hungarian Szeged](https://universaldependencies.org/treebanks/hu_szeged/index.html) (Richárd Farkas, Katalin Simkó, Zsolt Szántó, Viktor Varga, Veronika Vincze (MTA-SZTE Research Group on Artificial Intelligence))<br />[NYTK-NerKor Corpus](https://github.com/nytud/NYTK-NerKor) (Eszter Simon, Noémi Vadász (Department of Language Technology and Applied Linguistics))<br />[hunNERwiki](http://hlt.sztaki.hu/resources/hunnerwiki.html) (Eszter Simon, Dávid Márk Nemeskey (HLT Group, Budapest University of Technology and Economics))<br />[Szeged NER Corpus](https://rgai.inf.u-szeged.hu/node/130) (György Szarvas, Richárd Farkas, László Felföldi, András Kocsor, János Csirik (MTA-SZTE Research Group on Artificial Intelligence))<br />[huBERT base model (cased)](https://huggingface.co/SZTAKI-HLT/hubert-base-cc) (Dávid Márk Nemeskey (SZTAKI-HLT)) |
 | **License** | `cc-by-sa-4.0` |
@@ -118,11 +118,11 @@ Hungarian transformer pipeline (huBert) for HuSpaCy. Components: transformer, se
 | `MORPH_MICRO_P` | 97.75 |
 | `MORPH_MICRO_R` | 97.09 |
 | `MORPH_MICRO_F` | 97.42 |
-| `LEMMA_ACC` | 98.63 |
+| `LEMMA_ACC` | 98.64 |
 | `BOUND_DEP_LAS` | 87.15 |
 | `BOUND_DEP_UAS` | 91.19 |
 | `DEP_UAS` | 91.17 |
 | `DEP_LAS` | 87.13 |
-| `ENTS_P` | 90.78 |
-| `ENTS_R` | 91.42 |
-| `ENTS_F` | 91.10 |
+| `ENTS_P` | 90.95 |
+| `ENTS_R` | 90.79 |
+| `ENTS_F` | 90.87 |

config.cfg

@@ -1,7 +1,8 @@
 [paths]
-tagger_model = "models/hu_core_news_trf-tagger-3.2.1/model-best"
-parser_model = "models/hu_core_news_trf-parser-3.2.1/model-best"
-ner_model = "models/hu_core_news_trf-ner-3.2.1/model-best"
+tagger_model = "models/hu_core_news_trf-tagger-3.2.2/model-best"
+parser_model = "models/hu_core_news_trf-parser-3.2.2/model-best"
+ner_model = "models/hu_core_news_trf-ner-3.2.2/model-best"
+lemmatizer_lookups = "models/hu_core_news_trf-lookup-lemmatizer-3.2.2"
 train = null
 dev = null
 vectors = null
@@ -13,7 +14,7 @@ gpu_allocator = null
 
 [nlp]
 lang = "hu"
-pipeline = ["transformer","senter","tagger","morphologizer","experimental_edit_tree_lemmatizer","experimental_arc_predicter","experimental_arc_labeler","ner"]
+pipeline = ["transformer","senter","tagger","morphologizer","lookup_lemmatizer","experimental_edit_tree_lemmatizer","lemma_smoother","experimental_arc_predicter","experimental_arc_labeler","ner"]
 tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
 disabled = []
 before_creation = null
@@ -103,6 +104,14 @@ grad_factor = 1.0
 upstream = "transformer"
 pooling = {"@layers":"reduce_mean.v1"}
 
+[components.lemma_smoother]
+factory = "hu.lemma_smoother"
+
+[components.lookup_lemmatizer]
+factory = "hu.lookup_lemmatizer"
+scorer = {"@scorers":"spacy.lemmatizer_scorer.v1"}
+source = ${paths.lemmatizer_lookups}
+
 [components.morphologizer]
 factory = "morphologizer"
 extend = false

experimental_arc_labeler/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:990bd725ef174d1d6d38f22c6da304252fe49c5bb3a007727f65a38d38a145f5
+oid sha256:d9fc521c6aeb9d827e2439456aa44b28d4bd49845762e1ceda7d98e486a9e34d
 size 447476722

experimental_arc_predicter/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df945fcedd45cf4f83bea09366a04cf51e8a8bcc9409033108ae86e0c9cc0b18
+oid sha256:0c866a73a8ee7d3d8d53c8b6dc57b7929f16b501585cc0c6a15b06dc4b739dac
 size 445185682

hu_core_news_trf-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca3e08013b4549af474c26465027b678cdf3ec2edbeb3d2dc3a2715f3e13aa96
-size 1672216913
+oid sha256:75d7e1530fc472e1de19b73adb3da3cac98175a0c93498b423ee790aa4e220e6
+size 1674182137

lookup_lemmatizer/lookups.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7aaa1cfd45a0afd57ada8ccc690ee182485a151bdb133b7b8c9276647ab9e60
+size 2745978

meta.json

@@ -1,7 +1,7 @@
 {
   "lang":"hu",
   "name":"core_news_trf",
-  "version":"3.2.1",
+  "version":"3.2.2",
   "description":"Hungarian transformer pipeline (huBert) for HuSpaCy. Components: transformer, senter, tagger, morphologizer, lemmatizer, parser, ner",
   "author":"SzegedAI, MILAB",
   "email":"gyorgy@orosz.link",
@@ -1188,6 +1188,9 @@
       "Case=Dat|Number=Plur|POS=PRON|Person=1|PronType=Prs",
       "Case=Acc|Number=Plur|Number[psor]=Sing|POS=PROPN|Person[psor]=3",
       "Case=All|Number=Sing|Number[psed]=Sing|POS=PRON|Person=3|PronType=Tot"
+    ],
+    "lookup_lemmatizer":[
+
     ],
     "experimental_edit_tree_lemmatizer":[
       0,
@@ -7282,7 +7285,9 @@
     "senter",
     "tagger",
     "morphologizer",
+    "lookup_lemmatizer",
     "experimental_edit_tree_lemmatizer",
+    "lemma_smoother",
     "experimental_arc_predicter",
     "experimental_arc_labeler",
     "ner"
@@ -7292,7 +7297,9 @@
     "senter",
     "tagger",
     "morphologizer",
+    "lookup_lemmatizer",
     "experimental_edit_tree_lemmatizer",
+    "lemma_smoother",
     "experimental_arc_predicter",
     "experimental_arc_labeler",
     "ner"
@@ -7421,7 +7428,7 @@
         "f":0.8
       }
     },
-    "lemma_acc":0.9863170988,
+    "lemma_acc":0.9864127835,
     "bound_dep_las":0.8715420695,
     "bound_dep_uas":0.9119364411,
     "dep_uas":0.911718264,
@@ -7653,32 +7660,32 @@
         "f":0.0
       }
     },
-    "ents_p":0.9078212291,
-    "ents_r":0.9142053446,
-    "ents_f":0.9110021023,
+    "ents_p":0.9094751673,
+    "ents_r":0.9078762307,
+    "ents_f":0.9086749956,
     "ents_per_type":{
       "ORG":{
-        "p":0.9280742459,
-        "r":0.9272137228,
-        "f":0.9276437848
+        "p":0.9038718291,
+        "r":0.9415855355,
+        "f":0.9223433243
       },
       "PER":{
-        "p":0.9423648247,
-        "r":0.9474313023,
-        "f":0.944891272
+        "p":0.9425981873,
+        "r":0.9318996416,
+        "f":0.9372183839
       },
       "LOC":{
-        "p":0.9258936356,
-        "r":0.921875,
-        "f":0.9238799478
+        "p":0.9434822242,
+        "r":0.8984375,
+        "f":0.9204090707
       },
       "MISC":{
-        "p":0.7429340511,
-        "r":0.7829787234,
-        "f":0.7624309392
+        "p":0.7923416789,
+        "r":0.7631205674,
+        "f":0.7774566474
       }
     },
-    "speed":2488.2218404265
+    "speed":3603.1106108543
   },
   "sources":[
     {

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7965ae35085d883e8e9f9a5c5df11d763b93af32932f0c5a6e6040c49035fd4c
+oid sha256:3e513721ef78ad46ad0a39817bf0b82dd7b3ca2273155680f9cc6696f092d2a1
 size 443626204

transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9f20173c2a1ea1f3d4e540eb5fc7a72daa8844c31a0bd74d9c76c83c054df90
+oid sha256:ef1e9f39f0358ae8d9c4dc8e1427fc4ecf054e284ac5bba614cf9194266e2914
 size 443344004