update app structure + add new models to requirements

.gitignore

@@ -1,3 +1,4 @@
 Legacy.py
 .idea
-standoffconverter
+standoffconverter
+venv/

app.py

@@ -1,10 +1,10 @@
+import re
+
 import streamlit
 import spacy_streamlit
 import spacy
 from lxml import etree
 import pandas as pd
-from spacy import Language
-from spacy.tokens import Doc
 
 streamlit.set_page_config(layout="wide")
 
@@ -41,20 +41,12 @@ streamlit.write("## 📄 Input XML EAD:")
 filename = streamlit.file_uploader("Upload an XML EAD", type="xml")
 streamlit.markdown("or use an XML EAD provided in [`samples/`](https://huggingface.co/spaces/ner4archives/ner4archives-NEL-vizualizer-app/blob/main/samples/) directory")
 data = ""
-
-
-
-
 flag_model = False
+
 if filename is not None:
     data = filename.getvalue().decode("utf-8").encode("utf-8")
     if len(data) > 0:
         flag_file = True
-
-
-
-
-import re
 def ead_strategy(tree):
     # create a container for sentences and dids
     # elements
@@ -112,48 +104,7 @@ if flag_view:
         linkingicon = "❌"
     streamlit.write("#### Actual Parameters:")
     streamlit.write(f'- NER model selected: {option}\n - linking: {linkingicon}')
-@Language.factory("custom_ner", default_config={
-    "model_name": "",
-    "sentences_to_process": []
-})
-class CustomNer:
-    def __init__(self,
-                 nlp: Language,
-                 name: str,
-                 model_name: str,
-                 sentences_to_process: list):
-        self.nlp = nlp
-        self.pipeline_ner = spacy.load(model_name)
-        f_score = self.pipeline_ner.meta['performance']['ents_f']
-        recall  = self.pipeline_ner.meta['performance']['ents_r']
-        precision = self.pipeline_ner.meta['performance']['ents_p']
-        mcol1, mcol2, mcol3 = streamlit.columns(3)
-        mcol1.metric("F-Score", f'{f_score:.2f}')
-        mcol2.metric("Precision", f'{precision:.2f}')
-        mcol3.metric("Recall", f'{recall:.2f}')
-        self.sentences = sentences_to_process
-
-    def __call__(self, doc: Doc):
-        start_sentence = 0
-        spans = []
-        count = 0
-        bar = streamlit.progress(count)
-        for sent in self.pipeline_ner.pipe(self.sentences):
-            # add 1 char that correspond to space added in
-            # sentences concatenation (" ".join())
-            end_sentence = start_sentence + len(sent.text) + 1
-            # recompute named entities characters offsets
-            for ent in sent.ents:
-                start = start_sentence + ent.start_char
-                end = start + len(ent.text)
-                spans.append(doc.char_span(start, end, label=ent.label_))
-            start_sentence = end_sentence
-            count += 1
-            bar.progress((count/len(sentences))*1.0)
-
-        doc.set_ents(spans)
-
-        return doc
+
 
 entities = []
 docs = []
@@ -163,27 +114,44 @@ flag_vizualize = False
 # Launch NER process:
 if flag_model:
     if streamlit.button('Launch'):
+        plain = "\n".join(sentences)
         with streamlit.spinner('Initialize NER...'):
-
-            huge_pipeline_linking = spacy.blank("fr")
-            huge_pipeline_linking.max_length = 5000000
-            huge_pipeline_linking.add_pipe('custom_ner', config={"model_name": model, "sentences_to_process": sentences})
+            nlp = spacy.load(model)
+            nlp.max_length = 5000000
             if linking:
-                huge_pipeline_linking.add_pipe('entityfishing', config={"language": "fr"})
-                
+                nlp.add_pipe('entityfishing', config={"language": "fr"})
+
         with streamlit.spinner('NER processing...'):
-            doc = huge_pipeline_linking(plain)
-            #doc = huge_pipeline_linking(plain)
-
-        entities = [
-            (ent.start_char,
-            ent.end_char,
-            ent.text,
-            ent.label_,
-            ent._.url_wikidata if linking else None,
-             ent._.nerd_score if linking else None
-            ) for ent in doc.ents
-        ]
+            if linking:
+                start_sentence = 0
+                docs = nlp.pipe(sentences, batch_size=100)
+                for doc in docs:
+                    end_sentence = start_sentence + len(doc.text) + 1
+                    for ent in doc.ents:
+                        start_tok = start_sentence + ent.start_char
+                        end_tok = start_tok + len(ent.text)
+                        entities.append((
+                            start_tok,
+                            end_tok,
+                            ent.text,
+                            ent.label_,
+                            ent._.kb_qid,
+                            ent._.url_wikidata,
+                            ent._.nerd_score
+                        ))
+                    start_sentence = end_sentence
+            else:
+                entities = [
+                    (ent.start_char,
+                     ent.end_char,
+                     ent.text,
+                     ent.label_,
+                     "",
+                     "",
+                     ""
+                     ) for ent in nlp(plain).ents
+                ]
+
 
         streamlit.success('😃 NER applied with success!')
 
@@ -192,6 +160,7 @@ if flag_model:
                                              'END',
                                              'MENTION',
                                              'NER LABEL',
+                                             'QID',
                                              'WIKIDATA RESSOURCE  (wikidata disambiguation)',
                                              'LINKING SCORE'
                                              ])
@@ -199,32 +168,33 @@ if flag_model:
         streamlit.write("## 🔎 Explore named entities in table: ")
         streamlit.write(df)
 
+
         streamlit.write("## 🔎 Explore named entities in text: ")
         spacy_streamlit.visualize_ner(
-            {"text": doc.text, "ents": [{"start": ent.start_char,
-                  "end": ent.end_char,
-                  "label": ent.label_,
-                  "kb_id": ent._.kb_qid if linking else None,
-                  "kb_url": ent._.url_wikidata if linking else None
-                  } for ent in doc.ents]},
+            {"text": plain,
+             "ents": [{"start": ent[0],
+                  "end": ent[1],
+                  "label": ent[3],
+                  "kb_id": ent[4] if linking else "",
+                  "kb_url": ent[5] if linking else ""
+                  } for ent in entities]},
             labels=["EVENT", "LOCATION", "ORGANISATION", "PERSON", "TITLE", 'LOC', 'MISC', 'ORG', 'PER'],
             show_table=False,
             manual=True,
             title="",
-        displacy_options={
-            "colors": {
-                "EVENT": "#ec7063",
-                "LOCATION": "#45b39d",
-                "ORGANISATION": "#f39c12",
-                "PERSON": "#3498db",
-                "TITLE": "#a569bd ",
-                "LOC": "#45b39d",
-                "MISC": "#ec7063",
-                "ORG": "#f39c12",
-                "PER": "#3498db"
-
-            }
-        })
-   
+            displacy_options={
+                "colors": {
+                 "EVENT": "#ec7063",
+                 "LOCATION": "#45b39d",
+                 "ORGANISATION": "#f39c12",
+                 "PERSON": "#3498db",
+                 "TITLE": "#a569bd ",
+                 "LOC": "#45b39d",
+                 "MISC": "#ec7063",
+                 "ORG": "#f39c12",
+                 "PER": "#3498db"
+                }
+            })
+
 
 

models.md

@@ -0,0 +1,10 @@
+### Models available for Ner4Archives:
+
+* Corpus V2:
+  - fr_ner4archives_default_test @ https://huggingface.co/ner4archives/fr_ner4archives_default_test/resolve/main/fr_ner4archives_default_test-any-py3-none-any.whl
+  - fr_ner4archives_camembert_base @ https://huggingface.co/ner4archives/fr_ner4archives_camembert_base/resolve/main/fr_ner4archives_camembert_base-any-py3-none-any.whl
+
+* Corpus V3:
+  - fr_core_ner4archives_v3_default @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_default/resolve/main/fr_core_ner4archives_v3_default-any-py3-none-any.whl
+  - fr_core_ner4archives_v3_with_vectors @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_with_vectors/resolve/main/fr_core_ner4archives_v3_with_vectors-any-py3-none-any.whl
+  - fr_core_ner4archives_V3_camembert_base @ https://huggingface.co/ner4archives/fr_core_ner4archives_V3_camembert_base/resolve/main/fr_core_ner4archives_V3_camembert_base-any-py3-none-any.whl

requirements.txt

@@ -23,10 +23,6 @@ defusedxml==0.7.1
 entrypoints==0.4
 executing==0.9.1
 fastjsonschema==2.16.1
-fr-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.3.0/fr_core_news_sm-3.3.0-py3-none-any.whl
-fr-ner4archives-default-test @ https://huggingface.co/ner4archives/fr_ner4archives_default_test/resolve/main/fr_ner4archives_default_test-any-py3-none-any.whl
-fr_ner4archives_default_vectors_lg @ https://huggingface.co/ner4archives/fr_ner4archives_default_vectors_lg/resolve/main/fr_ner4archives_default_vectors_lg-any-py3-none-any.whl
-fr_ner4archives_camembert_base @ https://huggingface.co/ner4archives/fr_ner4archives_camembert_base/resolve/main/fr_ner4archives_camembert_base-any-py3-none-any.whl
 gitdb==4.0.9
 GitPython==3.1.27
 idna==3.3
@@ -90,16 +86,16 @@ six==1.16.0
 smart-open==5.2.1
 smmap==5.0.0
 soupsieve==2.3.2.post1
-spacy==3.3.1
+spacy==3.4.0
 spacy-legacy==3.0.9
 spacy-loggers==1.0.3
 spacy-streamlit==1.0.4
-spacyfishing==0.1.7
+spacyfishing==0.1.8
 srsly==2.4.4
 stack-data==0.3.0
 streamlit==1.11.1
 terminado==0.15.0
-thinc==8.0.17
+thinc==8.1.2
 tinycss2==1.1.1
 toml==0.10.2
 toolz==0.12.0
@@ -118,3 +114,6 @@ wcwidth==0.2.5
 webencodings==0.5.1
 widgetsnbextension==3.6.1
 zipp==3.8.1
+fr_core_ner4archives_v3_default @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_default/resolve/main/fr_core_ner4archives_v3_default-any-py3-none-any.whl
+fr_core_ner4archives_v3_with_vectors @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_with_vectors/resolve/main/fr_core_ner4archives_v3_with_vectors-any-py3-none-any.whl
+fr_core_ner4archives_V3_camembert_base @ https://huggingface.co/ner4archives/fr_core_ner4archives_V3_camembert_base/resolve/main/fr_core_ner4archives_V3_camembert_base-any-py3-none-any.whl