update batch size + endpoint EF

.gitignore

@@ -2,3 +2,4 @@ Legacy.py
 .idea
 standoffconverter
 venv/
+test.py

app.py

@@ -1,4 +1,5 @@
 import re
+import json
 
 import streamlit
 import spacy_streamlit
@@ -10,6 +11,11 @@ streamlit.set_page_config(layout="wide")
 
 samples_test = {"FRAN_IR_050370.xml": "./samples/FRAN_IR_050370.xml"}
 
+with open('config.json', mode="r") as json_file:
+    CONFIGURATION = json.loads(json_file.read())
+
+
+
 # TITLE APP
 streamlit.title("NER4Archives visualizer")
 streamlit.sidebar.title("NER4Archives visualizer")
@@ -119,13 +125,12 @@ if flag_model:
             nlp = spacy.load(model)
             nlp.max_length = 5000000
             if linking:
-                nlp.add_pipe('entityfishing', config={"language": "fr"})
+                nlp.add_pipe('entityfishing', config={"language": "fr", "api_ef_base": CONFIGURATION['ef_endpoint']})
 
         with streamlit.spinner('NER processing...'):
             if linking:
                 start_sentence = 0
-                docs = nlp.pipe(sentences, batch_size=150)
-                for doc in docs:
+                for doc in nlp.pipe(sentences, batch_size=250):
                     end_sentence = start_sentence + len(doc.text) + 1
                     for ent in doc.ents:
                         start_tok = start_sentence + ent.start_char
@@ -141,16 +146,21 @@ if flag_model:
                         ))
                     start_sentence = end_sentence
             else:
-                entities = [
-                    (ent.start_char,
-                     ent.end_char,
-                     ent.text,
-                     ent.label_,
-                     "",
-                     "",
-                     ""
-                     ) for ent in nlp(plain).ents
-                ]
+                start_sentence = 0
+                for doc in nlp.pipe(sentences):
+                    end_sentence = start_sentence + len(doc.text) + 1
+                    for ent in doc.ents:
+                        start_tok = start_sentence + ent.start_char
+                        end_tok = start_tok + len(ent.text)
+                        entities.append((start_tok,
+                                          end_tok,
+                                          ent.text,
+                                          ent.label_,
+                                          "",
+                                          "",
+                                          ""
+                                          ))
+                    start_sentence = end_sentence
 
 
         streamlit.success('😃 NER applied with success!')
@@ -194,7 +204,4 @@ if flag_model:
                  "ORG": "#f39c12",
                  "PER": "#3498db"
                 }
-            })
-
-
-
+            })

config.json

@@ -0,0 +1,3 @@
+{
+  "ef_endpoint": "http://nerd.huma-num.fr/nerd/service"
+}

requirements.txt

@@ -1,119 +1,8 @@
-altair==4.2.0
-argon2-cffi==21.3.0
-argon2-cffi-bindings==21.2.0
-asttokens==2.0.5
-attrs==22.1.0
-backcall==0.2.0
-backports.zoneinfo==0.2.1
-beautifulsoup4==4.11.1
-bleach==5.0.1
-blinker==1.5
-blis==0.7.8
-cachetools==5.2.0
-catalogue==2.0.8
-certifi==2022.6.15
-cffi==1.15.1
-charset-normalizer==2.1.0
-click==8.1.3
-commonmark==0.9.1
-cymem==2.0.6
-debugpy==1.6.2
-decorator==5.1.1
-defusedxml==0.7.1
-entrypoints==0.4
-executing==0.9.1
-fastjsonschema==2.16.1
-gitdb==4.0.9
-GitPython==3.1.27
-idna==3.3
-importlib-metadata==4.12.0
-importlib-resources==5.9.0
-ipykernel==6.15.1
-ipython==8.4.0
-ipython-genutils==0.2.0
-ipywidgets==7.7.1
-jedi==0.18.1
-Jinja2==3.1.2
-jsonschema==4.8.0
-jupyter-client==7.3.4
-jupyter-core==4.11.1
-jupyterlab-pygments==0.2.2
-jupyterlab-widgets==1.1.1
-langcodes==3.3.0
-lxml==4.9.1
-MarkupSafe==2.1.1
-matplotlib-inline==0.1.3
-mistune==0.8.4
-murmurhash==1.0.7
-nbclient==0.6.6
-nbconvert==6.5.0
-nbformat==5.4.0
-nest-asyncio==1.5.5
-notebook==6.4.12
-numpy==1.23.1
-packaging==21.3
-pandas==1.4.3
-pandocfilters==1.5.0
-parso==0.8.3
-pathy==0.6.2
-pexpect==4.8.0
-pickleshare==0.7.5
-Pillow==9.2.0
-preshed==3.0.6
-prometheus-client==0.14.1
-prompt-toolkit==3.0.30
-protobuf==3.20.1
-psutil==5.9.1
-ptyprocess==0.7.0
-pure-eval==0.2.2
-pyarrow==8.0.0
-pycparser==2.21
-pydantic==1.8.2
-pydeck==0.7.1
-Pygments==2.12.0
-Pympler==1.0.1
-pyparsing==3.0.9
-pyrsistent==0.18.1
-python-dateutil==2.8.2
-pytz==2022.1
-pytz-deprecation-shim==0.1.0.post0
-pyzmq==23.2.0
-requests==2.28.1
-rich==12.5.1
-semver==2.13.0
-Send2Trash==1.8.0
-six==1.16.0
-smart-open==5.2.1
-smmap==5.0.0
-soupsieve==2.3.2.post1
-spacy==3.4.0
-spacy-legacy==3.0.9
-spacy-loggers==1.0.3
+spacy==3.4.1
 spacy-streamlit==1.0.4
 spacyfishing==0.1.8
-srsly==2.4.4
-stack-data==0.3.0
 streamlit==1.11.1
-terminado==0.15.0
-thinc==8.1.2
-tinycss2==1.1.1
-toml==0.10.2
-toolz==0.12.0
-tornado==6.2
-tqdm==4.64.0
-traitlets==5.3.0
-typer==0.4.2
-typing_extensions==4.3.0
-tzdata==2022.1
-tzlocal==4.2
-urllib3==1.26.11
-validators==0.20.0
-wasabi==0.10.1
-watchdog==2.1.9
-wcwidth==0.2.5
-webencodings==0.5.1
-widgetsnbextension==3.6.1
-zipp==3.8.1
-fr_core_ner4archives_v3_default @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_default/resolve/main/fr_core_ner4archives_v3_default-any-py3-none-any.whl
-fr_core_ner4archives_v3_with_vectors @ https://huggingface.co/ner4archives/fr_core_ner4archives_v3_with_vectors/resolve/main/fr_core_ner4archives_v3_with_vectors-any-py3-none-any.whl
-fr_core_ner4archives_V3_camembert_base @ https://huggingface.co/ner4archives/fr_core_ner4archives_V3_camembert_base/resolve/main/fr_core_ner4archives_V3_camembert_base-any-py3-none-any.whl
+lxml==4.9.1
+spacy-transformers==1.1.8
+fr_ner4archives_v3_default @ https://huggingface.co/ner4archives/fr_ner4archives_v3_default/resolve/main/fr_ner4archives_v3_default-any-py3-none-any.whl
+fr_ner4archives_v3_with_vectors @ https://huggingface.co/ner4archives/fr_ner4archives_v3_with_vectors/resolve/main/fr_ner4archives_v3_with_vectors-any-py3-none-any.whl