Spaces:
Sleeping
Sleeping
langdonholmes
commited on
Commit
•
78706f0
1
Parent(s):
51c60c8
make anonymize importable without pydantic models
Browse files- piilo/__init__.py +1 -0
- piilo/engines/analyzer.py +12 -4
- piilo/main.py +14 -7
piilo/__init__.py
CHANGED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
from .main import anonymize
|
piilo/engines/analyzer.py
CHANGED
@@ -31,10 +31,16 @@ class CustomSpacyRecognizer(LocalRecognizer):
|
|
31 |
ner_strength: float = 0.85,
|
32 |
):
|
33 |
self.ner_strength = ner_strength
|
|
|
34 |
self.check_label_groups = (
|
35 |
-
check_label_groups if check_label_groups
|
|
|
36 |
)
|
37 |
-
supported_entities =
|
|
|
|
|
|
|
|
|
38 |
super().__init__(
|
39 |
supported_entities=supported_entities,
|
40 |
supported_language=supported_language,
|
@@ -88,7 +94,8 @@ class CustomSpacyRecognizer(LocalRecognizer):
|
|
88 |
if entity not in self.supported_entities:
|
89 |
continue
|
90 |
for ent in ner_entities:
|
91 |
-
if not self.__check_label(entity, ent.label_,
|
|
|
92 |
continue
|
93 |
textual_explanation = self.DEFAULT_EXPLANATION.format(
|
94 |
ent.label_)
|
@@ -114,7 +121,8 @@ class CustomSpacyRecognizer(LocalRecognizer):
|
|
114 |
entity: str, label: str, check_label_groups: Tuple[Set, Set]
|
115 |
) -> bool:
|
116 |
return any(
|
117 |
-
[entity in egrp and label in lgrp
|
|
|
118 |
)
|
119 |
|
120 |
class CustomAnalyzer(AnalyzerEngine):
|
|
|
31 |
ner_strength: float = 0.85,
|
32 |
):
|
33 |
self.ner_strength = ner_strength
|
34 |
+
|
35 |
self.check_label_groups = (
|
36 |
+
check_label_groups if check_label_groups
|
37 |
+
else self.CHECK_LABEL_GROUPS
|
38 |
)
|
39 |
+
supported_entities = (
|
40 |
+
supported_entities if supported_entities
|
41 |
+
else self.ENTITIES
|
42 |
+
)
|
43 |
+
|
44 |
super().__init__(
|
45 |
supported_entities=supported_entities,
|
46 |
supported_language=supported_language,
|
|
|
94 |
if entity not in self.supported_entities:
|
95 |
continue
|
96 |
for ent in ner_entities:
|
97 |
+
if not self.__check_label(entity, ent.label_,
|
98 |
+
self.check_label_groups):
|
99 |
continue
|
100 |
textual_explanation = self.DEFAULT_EXPLANATION.format(
|
101 |
ent.label_)
|
|
|
121 |
entity: str, label: str, check_label_groups: Tuple[Set, Set]
|
122 |
) -> bool:
|
123 |
return any(
|
124 |
+
[entity in egrp and label in lgrp
|
125 |
+
for egrp, lgrp in check_label_groups]
|
126 |
)
|
127 |
|
128 |
class CustomAnalyzer(AnalyzerEngine):
|
piilo/main.py
CHANGED
@@ -37,16 +37,23 @@ app.add_middleware(
|
|
37 |
def hello():
|
38 |
return {"message": "Hello World"}
|
39 |
|
40 |
-
|
41 |
-
def anonymize(anon_req: AnonymizeRequest) -> AnonymizeResponse:
|
42 |
|
43 |
-
analyzer_result = analyzer.analyze(
|
44 |
-
entities=
|
45 |
-
language=
|
46 |
)
|
47 |
|
48 |
-
|
49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
50 |
|
51 |
anonymize_response = AnonymizeResponse(
|
52 |
anonymized_text=anonymizer_result
|
|
|
37 |
def hello():
|
38 |
return {"message": "Hello World"}
|
39 |
|
40 |
+
def anonymize(raw_text: str, entities=None, language='en') -> str:
|
|
|
41 |
|
42 |
+
analyzer_result = analyzer.analyze(raw_text,
|
43 |
+
entities=entities,
|
44 |
+
language=language,
|
45 |
)
|
46 |
|
47 |
+
return anonymizer.anonymize(raw_text,
|
48 |
+
analyzer_result)
|
49 |
+
|
50 |
+
@app.post("/anonymize")
|
51 |
+
def get_anonymize(anon_req: AnonymizeRequest) -> AnonymizeResponse:
|
52 |
+
|
53 |
+
anonymizer_result = anonymize(anon_req.raw_text,
|
54 |
+
entities=anon_req.entities,
|
55 |
+
language=anon_req.language,
|
56 |
+
)
|
57 |
|
58 |
anonymize_response = AnonymizeResponse(
|
59 |
anonymized_text=anonymizer_result
|