Spaces:
Runtime error
Runtime error
feat: speed up demo loading time by using the common NLP object
Browse files- examples/anon.py +29 -19
examples/anon.py
CHANGED
@@ -1,26 +1,26 @@
|
|
1 |
from typing import Tuple, List
|
2 |
|
3 |
import gradio as gr
|
4 |
-
|
5 |
from presidio_analyzer import AnalyzerEngine
|
6 |
-
from presidio_analyzer.nlp_engine import
|
7 |
from presidio_anonymizer import AnonymizerEngine
|
8 |
-
|
9 |
-
from faker import Faker
|
10 |
from presidio_anonymizer.entities.engine import OperatorConfig
|
|
|
11 |
|
|
|
12 |
|
13 |
-
def process(text: str, fake_data: bool, entities: List) -> Tuple[str, List]:
|
14 |
-
configuration = {
|
15 |
-
"nlp_engine_name": "spacy",
|
16 |
-
"models": [{"lang_code": "hu", "model_name": "hu_core_news_trf", }],
|
17 |
-
}
|
18 |
|
19 |
-
|
20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
21 |
|
22 |
-
analyzer = AnalyzerEngine(nlp_engine=nlp_engine,
|
23 |
-
supported_languages=["hu"],)
|
24 |
|
25 |
results = analyzer.analyze(
|
26 |
text=text, entities=entities, language="hu")
|
@@ -42,17 +42,25 @@ def process(text: str, fake_data: bool, entities: List) -> Tuple[str, List]:
|
|
42 |
|
43 |
anonymizer = AnonymizerEngine()
|
44 |
anonymized_text = anonymizer.anonymize(
|
45 |
-
text=text, analyzer_results=results, operators=fake_operators) if fake_data else anonymizer.anonymize(text=text,
|
|
|
46 |
|
47 |
return anonymized_text.text, anonymized_text.items
|
48 |
|
49 |
|
50 |
EXAMPLES = [
|
51 |
-
[
|
52 |
-
|
|
|
|
|
|
|
|
|
53 |
"PERSON", "LOCATION"]],
|
54 |
-
[
|
55 |
-
|
|
|
|
|
|
|
56 |
]
|
57 |
|
58 |
demo = gr.Interface(
|
@@ -60,7 +68,9 @@ demo = gr.Interface(
|
|
60 |
inputs=[gr.Textbox(value=EXAMPLES[0][0], lines=10, label="Input text", show_label=True),
|
61 |
gr.Checkbox(value=EXAMPLES[0][1],
|
62 |
label="Apply de-identification", show_label=True),
|
63 |
-
gr.CheckboxGroup(
|
|
|
|
|
64 |
outputs=[gr.Textbox(label="Anonymized text", show_label=True),
|
65 |
gr.Textbox(label="Tags", show_label=True)],
|
66 |
examples=EXAMPLES,
|
|
|
1 |
from typing import Tuple, List
|
2 |
|
3 |
import gradio as gr
|
4 |
+
from faker import Faker
|
5 |
from presidio_analyzer import AnalyzerEngine
|
6 |
+
from presidio_analyzer.nlp_engine import SpacyNlpEngine
|
7 |
from presidio_anonymizer import AnonymizerEngine
|
|
|
|
|
8 |
from presidio_anonymizer.entities.engine import OperatorConfig
|
9 |
+
from spacy import Language
|
10 |
|
11 |
+
from examples.common import NLP
|
12 |
|
|
|
|
|
|
|
|
|
|
|
13 |
|
14 |
+
# noinspection PyMissingConstructor
|
15 |
+
class HuSpaCyNlpEngine(SpacyNlpEngine):
|
16 |
+
def __init__(self, nlp: Language):
|
17 |
+
self.nlp = {"hu": nlp}
|
18 |
+
|
19 |
+
|
20 |
+
def process(text: str, fake_data: bool, entities: List) -> Tuple[str, List]:
|
21 |
+
nlp_engine = HuSpaCyNlpEngine(NLP)
|
22 |
|
23 |
+
analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=["hu"])
|
|
|
24 |
|
25 |
results = analyzer.analyze(
|
26 |
text=text, entities=entities, language="hu")
|
|
|
42 |
|
43 |
anonymizer = AnonymizerEngine()
|
44 |
anonymized_text = anonymizer.anonymize(
|
45 |
+
text=text, analyzer_results=results, operators=fake_operators) if fake_data else anonymizer.anonymize(text=text,
|
46 |
+
analyzer_results=results)
|
47 |
|
48 |
return anonymized_text.text, anonymized_text.items
|
49 |
|
50 |
|
51 |
EXAMPLES = [
|
52 |
+
[
|
53 |
+
"Vespucci 1450-es években született Firenzében, és 1497 és 1504 között legalább két felfedező úton vett részt – az egyiket spanyol, a másikat portugál támogatással.",
|
54 |
+
False, ["PERSON", "LOCATION"]],
|
55 |
+
[
|
56 |
+
"Elon Musk 1971-ben született a Dél-afrikai Köztársaságban, anyja Maye Musk (született: Haldeman) modell, apja Errol Musk mérnök, pilóta.",
|
57 |
+
True, [
|
58 |
"PERSON", "LOCATION"]],
|
59 |
+
[
|
60 |
+
"Vespucci 1450-es években született Firenzében, és 1497 és 1504 között legalább két felfedező úton vett részt. Bárorító leveleket a vespucci@deojeda.es email-címre várt, mellette működött egy hangrögzítője is a +3903827802737 telefonszámon. Adományokat a bitcoin tárcájába (1Fsb3io3hj1jKaRCTRQ89Du88Dp7NxgEcU), bankkártyájára (5200 8282 8282 8210) és IBAN számlaszámára (ES8201289482186115378819) fogadott. Utazási blogja a https://firenze.it/vespucci címen volt elérhető. Legutóbb 1503-03-15-én publikált, ezt a 192.168.0.1 ip-címről tette meg.",
|
61 |
+
True,
|
62 |
+
["PERSON", "LOCATION", "EMAIL_ADDRESS", "PHONE_NUMBER", "CRYPTO", "IP_ADDRESS", "URL", "DATE_TIME",
|
63 |
+
"CREDIT_CARD", "IBAN_CODE"]],
|
64 |
]
|
65 |
|
66 |
demo = gr.Interface(
|
|
|
68 |
inputs=[gr.Textbox(value=EXAMPLES[0][0], lines=10, label="Input text", show_label=True),
|
69 |
gr.Checkbox(value=EXAMPLES[0][1],
|
70 |
label="Apply de-identification", show_label=True),
|
71 |
+
gr.CheckboxGroup(
|
72 |
+
['PERSON', 'LOCATION', 'DATE_TIME', 'IP_ADDRESS', 'URL', 'EMAIL_ADDRESS', 'PHONE_NUMBER', 'CREDIT_CARD',
|
73 |
+
'IBAN_CODE', 'CRYPTO'], label="Entities", show_label=True, value=EXAMPLES[0][2])],
|
74 |
outputs=[gr.Textbox(label="Anonymized text", show_label=True),
|
75 |
gr.Textbox(label="Tags", show_label=True)],
|
76 |
examples=EXAMPLES,
|