Add application file

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/build_config.yaml

@@ -0,0 +1,14 @@
+-
+    name: presidio_analyzer
+    version: 4.1.0
+    build: 0.0.1
+    author: Amit Hegde
+    author_email: amitumamaheshwar.h@infosys.com
+    description: Infosys Intelligent Assistant
+    long_description: Infosys Intelligent Assistant
+    classifiers: ["Programming Language :: Python :: 3",
+                  "License :: OSI Approved :: MIT License",
+                  "Operating System :: OS Independent",]
+    package_dir: {"": "presidio_analyzer"}
+    packages: presidio_analyzer
+    python_requires: ['>=3.6']

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/create_wheel_file.py

@@ -0,0 +1,44 @@
+__copyright__ = """ 2020 - 2021 Infosys Limited, Bangalore, India. All Rights Reserved.  
+Version: 2.5.0.0 
+Except for any free or open source software components embedded in this Infosys proprietary software program (“Program”), this Program is protected by copyright laws, international treaties and other pending or existing intellectual property rights in India, the United States and other countries.
+Except as expressly permitted, any unauthorized reproduction, storage, transmission in any form or by any means (including without limitation electronic, mechanical, printing, photocopying, recording or otherwise), or any distribution of this Program, or any portion of it, may result in severe civil and criminal penalties, and will be prosecuted to the maximum extent possible under the law.
+"""
+import yaml
+import subprocess
+import os
+with open(r'.\build_config.yaml') as build_file:
+    build_config_list = yaml.safe_load(build_file)
+
+
+for build_config in build_config_list:
+
+    try:
+        print(build_config)
+
+        if os.path.exists(f"./{build_config['packages']}"):
+
+            setup_str = f"import setuptools\r" \
+                        f"setuptools.setup(\r \
+                name='{build_config['name']}',\r \
+                version='{build_config['version']}',\r \
+                author='{build_config['author']}',\r \
+                author_email='{build_config['author_email']}',\r \
+                description='{build_config['description']}',\r \
+                long_description='{build_config['long_description']}',\r \
+                classifiers={build_config['classifiers']},\r \
+                package_dir={build_config['package_dir']},\r \
+                packages=setuptools.find_packages(where='{build_config['packages']}'),\r \
+                python_requires='{build_config['python_requires'][0]}',\r \
+                )"
+
+            with open('setup.py','w') as file:
+                file.write(setup_str)
+
+            subprocess.run(["python", "-m","build"])
+            wheel_file = f"{build_config['name']}-{build_config['version']}_build_{build_config['build']}-py3-none-any.whl"
+            print(f"wheel_file: {wheel_file}")
+            subprocess.run(["python", "-m", "pyc_wheel", f"dist\{wheel_file}"])
+        else:
+            print(f"Path does not exist ./{build_config['packages']}")
+    except Exception as e:
+        print(e)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.0.5.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:effdee5c88badc2a4605dcabc7fe1ff43df586df0a7c2be3f4dbc4d440c7e4d6
+size 44375

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.0.6.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49ca4ee3acda590bb69b68697e02cbfc81b89bd8dcfcaf9ff90b07fec062515
+size 44656

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.1.0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595ba3a58a473cc94a2a5c421eea075c5db52cb0181335a92f3a222f5cc76736
+size 44675

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: presidio_analyzer
+Version: 4.1.0
+Summary: Infosys Intelligent Assistant
+Author: Amit Hegde
+Author-email: amitumamaheshwar.h@infosys.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+
+Infosys Intelligent Assistant

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/SOURCES.txt

@@ -0,0 +1,64 @@
+setup.py
+presidio_analyzer/presidio_analyzer/__init__.py
+presidio_analyzer/presidio_analyzer/analysis_explanation.py
+presidio_analyzer/presidio_analyzer/analyzer_engine.py
+presidio_analyzer/presidio_analyzer/analyzer_request.py
+presidio_analyzer/presidio_analyzer/app_tracer.py
+presidio_analyzer/presidio_analyzer/batch_analyzer_engine.py
+presidio_analyzer/presidio_analyzer/dict_analyzer_result.py
+presidio_analyzer/presidio_analyzer/entity_recognizer.py
+presidio_analyzer/presidio_analyzer/local_recognizer.py
+presidio_analyzer/presidio_analyzer/pattern.py
+presidio_analyzer/presidio_analyzer/pattern_recognizer.py
+presidio_analyzer/presidio_analyzer/recognizer_result.py
+presidio_analyzer/presidio_analyzer/remote_recognizer.py
+presidio_analyzer/presidio_analyzer.egg-info/PKG-INFO
+presidio_analyzer/presidio_analyzer.egg-info/SOURCES.txt
+presidio_analyzer/presidio_analyzer.egg-info/dependency_links.txt
+presidio_analyzer/presidio_analyzer.egg-info/top_level.txt
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/__init__.py
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/context_aware_enhancer.py
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/lemma_context_aware_enhancer.py
+presidio_analyzer/presidio_analyzer/nlp_engine/__init__.py
+presidio_analyzer/presidio_analyzer/nlp_engine/client_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine_provider.py
+presidio_analyzer/presidio_analyzer/nlp_engine/spacy_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/Aadhaar_Number.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/PAN_Number.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/__init__.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_abn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_acn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_medicare_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_tfn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/credit_card_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/crypto_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/data_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/date_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/email_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/es_nif_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/iban_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/ip_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_driver_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_fiscal_code_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_identity_card_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_passport_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_vat_code.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/medical_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/phone_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/sg_fin_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/spacy_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/stanza_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/transformers_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/uk_nhs_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/url_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_driver_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_itin_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_passport_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_ssn_recognizer.py
+presidio_analyzer/presidio_analyzer/recognizer_registry/__init__.py
+presidio_analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/top_level.txt

@@ -0,0 +1 @@
+presidio_analyzer

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/__init__.py

@@ -0,0 +1,52 @@
+"""Presidio analyzer package."""
+
+import logging
+
+from presidio_analyzer.pattern import Pattern
+from presidio_analyzer.analysis_explanation import AnalysisExplanation
+from presidio_analyzer.recognizer_result import RecognizerResult
+from presidio_analyzer.dict_analyzer_result import DictAnalyzerResult
+from presidio_analyzer.entity_recognizer import EntityRecognizer
+from presidio_analyzer.local_recognizer import LocalRecognizer
+from presidio_analyzer.pattern_recognizer import PatternRecognizer
+from presidio_analyzer.remote_recognizer import RemoteRecognizer
+from presidio_analyzer.recognizer_registry import RecognizerRegistry
+from presidio_analyzer.analyzer_engine import AnalyzerEngine
+from presidio_analyzer.batch_analyzer_engine import BatchAnalyzerEngine
+from presidio_analyzer.analyzer_request import AnalyzerRequest
+from presidio_analyzer.context_aware_enhancers import ContextAwareEnhancer
+from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer
+
+
+# Define default loggers behavior
+
+# 1. presidio_analyzer logger
+
+logging.getLogger("presidio_analyzer").addHandler(logging.NullHandler())
+
+# 2. decision_process logger.
+# Setting the decision process trace here as we would want it
+# to be activated using a parameter to AnalyzeEngine and not by default.
+
+decision_process_logger = logging.getLogger("decision_process")
+ch = logging.StreamHandler()
+formatter = logging.Formatter("[%(asctime)s][%(name)s][%(levelname)s]%(message)s")
+ch.setFormatter(formatter)
+decision_process_logger.addHandler(ch)
+decision_process_logger.setLevel("INFO")
+__all__ = [
+    "Pattern",
+    "AnalysisExplanation",
+    "RecognizerResult",
+    "DictAnalyzerResult",
+    "EntityRecognizer",
+    "LocalRecognizer",
+    "PatternRecognizer",
+    "RemoteRecognizer",
+    "RecognizerRegistry",
+    "AnalyzerEngine",
+    "AnalyzerRequest",
+    "ContextAwareEnhancer",
+    "LemmaContextAwareEnhancer",
+    "BatchAnalyzerEngine",
+]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analysis_explanation.py

@@ -0,0 +1,64 @@
+from typing import Dict
+
+
+class AnalysisExplanation:
+    """
+    Hold tracing information to explain why PII entities were identified as such.
+
+    :param recognizer: name of recognizer that made the decision
+    :param original_score: recognizer's confidence in result
+    :param pattern_name: name of pattern
+            (if decision was made by a PatternRecognizer)
+    :param pattern: regex pattern that was applied (if PatternRecognizer)
+    :param validation_result: result of a validation (e.g. checksum)
+    :param textual_explanation: Free text for describing
+            a decision of a logic or model
+    """
+
+    def __init__(
+        self,
+        recognizer: str,
+        original_score: float,
+        pattern_name: str = None,
+        pattern: str = None,
+        validation_result: float = None,
+        textual_explanation: str = None,
+    ):
+
+        self.recognizer = recognizer
+        self.pattern_name = pattern_name
+        self.pattern = pattern
+        self.original_score = original_score
+        self.score = original_score
+        self.textual_explanation = textual_explanation
+        self.score_context_improvement = 0
+        self.supportive_context_word = ""
+        self.validation_result = validation_result
+
+    def __repr__(self):
+        """Create string representation of the object."""
+        return str(self.__dict__)
+
+    def set_improved_score(self, score: float) -> None:
+        """Update the score and calculate the difference from the original score."""
+        self.score = score
+        self.score_context_improvement = self.score - self.original_score
+
+    def set_supportive_context_word(self, word: str) -> None:
+        """Set the context word which helped increase the score."""
+        self.supportive_context_word = word
+
+    def append_textual_explanation_line(self, text: str) -> None:
+        """Append a new line to textual_explanation field."""
+        if self.textual_explanation is None:
+            self.textual_explanation = text
+        else:
+            self.textual_explanation = "{}\n{}".format(self.textual_explanation, text)
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return self.__dict__

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analyzer_engine.py

@@ -0,0 +1,372 @@
+import json
+import logging
+from typing import List, Optional
+
+from presidio_analyzer import (
+    RecognizerRegistry,
+    RecognizerResult,
+    EntityRecognizer,
+)
+from presidio_analyzer.app_tracer import AppTracer
+from presidio_analyzer.context_aware_enhancers import (
+    ContextAwareEnhancer,
+    LemmaContextAwareEnhancer,
+)
+from presidio_analyzer.nlp_engine import NlpEngine, NlpEngineProvider, NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class AnalyzerEngine:
+    """
+    Entry point for Presidio Analyzer.
+
+    Orchestrating the detection of PII entities and all related logic.
+
+    :param registry: instance of type RecognizerRegistry
+    :param nlp_engine: instance of type NlpEngine
+    (for example SpacyNlpEngine)
+    :param app_tracer: instance of type AppTracer, used to trace the logic
+    used during each request for interpretability reasons.
+    :param log_decision_process: bool,
+    defines whether the decision process within the analyzer should be logged or not.
+    :param default_score_threshold: Minimum confidence value
+    for detected entities to be returned
+    :param supported_languages: List of possible languages this engine could be run on.
+    Used for loading the right NLP models and recognizers for these languages.
+    :param context_aware_enhancer: instance of type ContextAwareEnhancer for enhancing
+    confidence score based on context words, (LemmaContextAwareEnhancer will be created
+    by default if None passed)
+    """
+
+    def __init__(
+        self,
+        registry: RecognizerRegistry = None,
+        nlp_engine: NlpEngine = None,
+        app_tracer: AppTracer = None,
+        log_decision_process: bool = False,
+        default_score_threshold: float = 0,
+        supported_languages: List[str] = None,
+        context_aware_enhancer: Optional[ContextAwareEnhancer] = None,
+    ):
+        if not supported_languages:
+            supported_languages = ["en"]
+
+        if not nlp_engine:
+            logger.info("nlp_engine not provided, creating default.")
+            provider = NlpEngineProvider()
+            nlp_engine = provider.create_engine()
+
+        if not registry:
+            logger.info("registry not provided, creating default.")
+            registry = RecognizerRegistry()
+        if not app_tracer:
+            app_tracer = AppTracer()
+        self.app_tracer = app_tracer
+
+        self.supported_languages = supported_languages
+
+        self.nlp_engine = nlp_engine
+        self.registry = registry
+
+        # load all recognizers
+        if not registry.recognizers:
+            registry.load_predefined_recognizers(
+                nlp_engine=self.nlp_engine, languages=self.supported_languages
+            )
+
+        self.log_decision_process = log_decision_process
+        self.default_score_threshold = default_score_threshold
+
+        if not context_aware_enhancer:
+            logger.debug(
+                "context aware enhancer not provided, creating default"
+                + " lemma based enhancer."
+            )
+            context_aware_enhancer = LemmaContextAwareEnhancer()
+
+        self.context_aware_enhancer = context_aware_enhancer
+
+    def get_recognizers(self, language: Optional[str] = None) -> List[EntityRecognizer]:
+        """
+        Return a list of PII recognizers currently loaded.
+
+        :param language: Return the recognizers supporting a given language.
+        :return: List of [Recognizer] as a RecognizersAllResponse
+        """
+        if not language:
+            languages = self.supported_languages
+        else:
+            languages = [language]
+
+        recognizers = []
+        for language in languages:
+            logger.info(f"Fetching all recognizers for language {language}")
+            recognizers.extend(
+                self.registry.get_recognizers(language=language, all_fields=True)
+            )
+
+        return list(set(recognizers))
+
+    def get_supported_entities(self, language: Optional[str] = None) -> List[str]:
+        """
+        Return a list of the entities that can be detected.
+
+        :param language: Return only entities supported in a specific language.
+        :return: List of entity names
+        """
+        recognizers = self.get_recognizers(language=language)
+        supported_entities = []
+        for recognizer in recognizers:
+            supported_entities.extend(recognizer.get_supported_entities())
+
+        return list(set(supported_entities))
+
+    def analyze(
+        self,
+        text: str,
+        language: str,
+        entities: Optional[List[str]] = None,
+        correlation_id: Optional[str] = None,
+        score_threshold: Optional[float] = None,
+        return_decision_process: Optional[bool] = False,
+        ad_hoc_recognizers: Optional[List[EntityRecognizer]] = None,
+        context: Optional[List[str]] = None,
+        allow_list: Optional[List[str]] = None,
+        nlp_artifacts: Optional[NlpArtifacts] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Find PII entities in text using different PII recognizers for a given language.
+
+        :param text: the text to analyze
+        :param language: the language of the text
+        :param entities: List of PII entities that should be looked for in the text.
+        If entities=None then all entities are looked for.
+        :param correlation_id: cross call ID for this request
+        :param score_threshold: A minimum value for which
+        to return an identified entity
+        :param return_decision_process: Whether the analysis decision process steps
+        returned in the response.
+        :param ad_hoc_recognizers: List of recognizers which will be used only
+        for this specific request.
+        :param context: List of context words to enhance confidence score if matched
+        with the recognized entity's recognizer context
+        :param allow_list: List of words that the user defines as being allowed to keep
+        in the text
+        :param nlp_artifacts: precomputed NlpArtifacts
+        :return: an array of the found entities in the text
+
+        :example:
+
+        >>> from presidio_analyzer import AnalyzerEngine
+
+        >>> # Set up the engine, loads the NLP module (spaCy model by default)
+        >>> # and other PII recognizers
+        >>> analyzer = AnalyzerEngine()
+
+        >>> # Call analyzer to get results
+        >>> results = analyzer.analyze(text='My phone number is 212-555-5555', entities=['PHONE_NUMBER'], language='en') # noqa D501
+        >>> print(results)
+        [type: PHONE_NUMBER, start: 19, end: 31, score: 0.85]
+        """
+        all_fields = not entities
+
+        recognizers = self.registry.get_recognizers(
+            language=language,
+            entities=entities,
+            all_fields=all_fields,
+            ad_hoc_recognizers=ad_hoc_recognizers,
+        )
+
+        if all_fields:
+            # Since all_fields=True, list all entities by iterating
+            # over all recognizers
+            entities = self.get_supported_entities(language=language)
+
+        # run the nlp pipeline over the given text, store the results in
+        # a NlpArtifacts instance
+        if not nlp_artifacts:
+            nlp_artifacts = self.nlp_engine.process_text(text, language)
+
+        if self.log_decision_process:
+            self.app_tracer.trace(
+                correlation_id, "nlp artifacts:" + nlp_artifacts.to_json()
+            )
+
+        results = []
+        for recognizer in recognizers:
+            # Lazy loading of the relevant recognizers
+            if not recognizer.is_loaded:
+                recognizer.load()
+                recognizer.is_loaded = True
+
+            # analyze using the current recognizer and append the results
+            current_results = recognizer.analyze(
+                text=text, entities=entities, nlp_artifacts=nlp_artifacts
+            )
+
+            if current_results:
+                # add recognizer name to recognition metadata inside results
+                # if not exists
+                self.__add_recognizer_id_if_not_exists(current_results, recognizer)
+                results.extend(current_results)
+
+
+        results = self._enhance_using_context(
+            text, results, nlp_artifacts, recognizers, context
+        )
+
+        if self.log_decision_process:
+            self.app_tracer.trace(
+                correlation_id,
+                json.dumps([str(result.to_dict()) for result in results]),
+            )
+
+        # Remove duplicates or low score results
+        results = EntityRecognizer.remove_duplicates(results)
+        results = self.__remove_low_scores(results, score_threshold)
+
+        if allow_list:
+            results = self._remove_allow_list(results, allow_list, text)
+
+        if not return_decision_process:
+            results = self.__remove_decision_process(results)
+
+        return results
+
+    def _enhance_using_context(
+        self,
+        text: str,
+        raw_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        recognizers: List[EntityRecognizer],
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Enhance confidence score using context words.
+
+        :param text: The actual text that was analyzed
+        :param raw_results: Recognizer results which didn't take
+                            context into consideration
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param recognizers: the list of recognizers
+        :param context: list of context words
+        """
+        results = []
+
+        for recognizer in recognizers:
+            recognizer_results = [
+                r
+                for r in raw_results
+                if r.recognition_metadata[RecognizerResult.RECOGNIZER_IDENTIFIER_KEY]
+                == recognizer.id
+            ]
+            other_recognizer_results = [
+                r
+                for r in raw_results
+                if r.recognition_metadata[RecognizerResult.RECOGNIZER_IDENTIFIER_KEY]
+                != recognizer.id
+            ]
+
+            # enhance score using context in recognizer level if implemented
+            recognizer_results = recognizer.enhance_using_context(
+                text=text,
+                # each recognizer will get access to all recognizer results
+                # to allow related entities contex enhancement
+                raw_recognizer_results=recognizer_results,
+                other_raw_recognizer_results=other_recognizer_results,
+                nlp_artifacts=nlp_artifacts,
+                context=context,
+            )
+
+            results.extend(recognizer_results)
+
+        # Update results in case surrounding words or external context are relevant to
+        # the context words.
+        results = self.context_aware_enhancer.enhance_using_context(
+            text=text,
+            raw_results=results,
+            nlp_artifacts=nlp_artifacts,
+            recognizers=recognizers,
+            context=context,
+        )
+
+        return results
+
+    def __remove_low_scores(
+        self, results: List[RecognizerResult], score_threshold: float = None
+    ) -> List[RecognizerResult]:
+        """
+        Remove results for which the confidence is lower than the threshold.
+
+        :param results: List of RecognizerResult
+        :param score_threshold: float value for minimum possible confidence
+        :return: List[RecognizerResult]
+        """
+        if score_threshold is None:
+            score_threshold = self.default_score_threshold
+
+        new_results = [result for result in results if result.score >= score_threshold]
+        return new_results
+
+    @staticmethod
+    def _remove_allow_list(
+        results: List[RecognizerResult], allow_list: List[str], text: str
+    ) -> List[RecognizerResult]:
+        """
+        Remove results which are part of the allow list.
+
+        :param results: List of RecognizerResult
+        :param allow_list: list of allowed terms
+        :param text: the text to analyze
+        :return: List[RecognizerResult]
+        """
+        new_results = []
+        for result in results:
+            word = text[result.start : result.end]
+            # if the word is not specified to be allowed, keep in the PII entities
+            if word not in allow_list:
+                new_results.append(result)
+
+        return new_results
+
+    @staticmethod
+    def __add_recognizer_id_if_not_exists(
+        results: List[RecognizerResult], recognizer: EntityRecognizer
+    ):
+        """Ensure recognition metadata with recognizer id existence.
+
+        Ensure recognizer result list contains recognizer id inside recognition
+        metadata dictionary, and if not create it. recognizer_id is needed
+        for context aware enhancement.
+
+        :param results: List of RecognizerResult
+        :param recognizer: Entity recognizer
+        """
+        for result in results:
+            if not result.recognition_metadata:
+                result.recognition_metadata = dict()
+            if (
+                RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                not in result.recognition_metadata
+            ):
+                result.recognition_metadata[
+                    RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                ] = recognizer.id
+            if RecognizerResult.RECOGNIZER_NAME_KEY not in result.recognition_metadata:
+                result.recognition_metadata[
+                    RecognizerResult.RECOGNIZER_NAME_KEY
+                ] = recognizer.name
+
+    @staticmethod
+    def __remove_decision_process(
+        results: List[RecognizerResult],
+    ) -> List[RecognizerResult]:
+        """Remove decision process / analysis explanation from response."""
+
+        for result in results:
+            result.analysis_explanation = None
+
+        return results

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analyzer_request.py

@@ -0,0 +1,36 @@
+from typing import Dict
+
+from presidio_analyzer import PatternRecognizer
+
+
+class AnalyzerRequest:
+    """
+    Analyzer request data.
+
+    :param req_data: A request dictionary with the following fields:
+        text: the text to analyze
+        language: the language of the text
+        entities: List of PII entities that should be looked for in the text.
+        If entities=None then all entities are looked for.
+        correlation_id: cross call ID for this request
+        score_threshold: A minimum value for which to return an identified entity
+        log_decision_process: Should the decision points within the analysis
+        be logged
+        return_decision_process: Should the decision points within the analysis
+        returned as part of the response
+    """
+
+    def __init__(self, req_data: Dict):
+        self.text = req_data.get("text")
+        self.language = req_data.get("language")
+        self.entities = req_data.get("entities")
+        self.correlation_id = req_data.get("correlation_id")
+        self.score_threshold = req_data.get("score_threshold")
+        self.return_decision_process = req_data.get("return_decision_process")
+        ad_hoc_recognizers = req_data.get("ad_hoc_recognizers")
+        self.ad_hoc_recognizers = []
+        if ad_hoc_recognizers:
+            self.ad_hoc_recognizers = [
+                PatternRecognizer.from_dict(rec) for rec in ad_hoc_recognizers
+            ]
+        self.context = req_data.get("context")

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/app_tracer.py

@@ -0,0 +1,27 @@
+import logging
+
+
+class AppTracer:
+    """
+    Allow logging/tracing the system's decisions.
+
+    Relevant in cases where we want to know which modules were used for detection,
+    which logic was utilized, what results were given and potentially why.
+    This can be useful for analyzing the detection accuracy of the system.
+    :param enabled: Whether tracing should be activated.
+    """
+
+    def __init__(self, enabled: bool = True):
+        self.logger = logging.getLogger("decision_process")
+        self.enabled = enabled
+
+    def trace(self, request_id: str, trace_data: str) -> None:
+        """
+        Write a value associated with a decision for a specific request into the trace.
+
+        Tracing for further inspection if needed.
+        :param request_id: A unique ID, to correlate across calls.
+        :param trace_data: A string to write to the log.
+        """
+        if self.enabled:
+            self.logger.info("[%s][%s]", request_id, trace_data)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/batch_analyzer_engine.py

@@ -0,0 +1,145 @@
+import logging
+from typing import List, Iterable, Dict, Union, Any, Optional, Iterator, Tuple
+
+from presidio_analyzer import DictAnalyzerResult, RecognizerResult, AnalyzerEngine
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class BatchAnalyzerEngine:
+    """
+    Batch analysis of documents (tables, lists, dicts).
+
+    Wrapper class to run Presidio Analyzer Engine on multiple values,
+    either lists/iterators of strings, or dictionaries.
+
+    :param: analyzer_engine: AnalyzerEngine instance to use
+    for handling the values in those collections.
+    """
+
+    def __init__(self, analyzer_engine: Optional[AnalyzerEngine] = None):
+
+        self.analyzer_engine = analyzer_engine
+        if not analyzer_engine:
+            self.analyzer_engine = AnalyzerEngine()
+
+    def analyze_iterator(
+        self,
+        texts: Iterable[Union[str, bool, float, int]],
+        language: str,
+        **kwargs,
+    ) -> List[List[RecognizerResult]]:
+        """
+        Analyze an iterable of strings.
+
+        :param texts: An list containing strings to be analyzed.
+        :param language: Input language
+        :param kwargs: Additional parameters for the `AnalyzerEngine.analyze` method.
+        """
+
+        # validate types
+        texts = self._validate_types(texts)
+
+        # Process the texts as batch for improved performance
+        nlp_artifacts_batch: Iterator[
+            Tuple[str, NlpArtifacts]
+        ] = self.analyzer_engine.nlp_engine.process_batch(
+            texts=texts, language=language
+        )
+
+        list_results = []
+        for text, nlp_artifacts in nlp_artifacts_batch:
+            results = self.analyzer_engine.analyze(
+                text=str(text), nlp_artifacts=nlp_artifacts, language=language, **kwargs
+            )
+
+            list_results.append(results)
+
+        return list_results
+
+    def analyze_dict(
+        self,
+        input_dict: Dict[str, Union[Any, Iterable[Any]]],
+        language: str,
+        keys_to_skip: Optional[List[str]] = None,
+        **kwargs,
+    ) -> Iterator[DictAnalyzerResult]:
+        """
+        Analyze a dictionary of keys (strings) and values/iterable of values.
+
+        Non-string values are returned as is.
+
+        :param input_dict: The input dictionary for analysis
+        :param language: Input language
+        :param keys_to_skip: Keys to ignore during analysis
+        :param kwargs: Additional keyword arguments
+        for the `AnalyzerEngine.analyze` method.
+        Use this to pass arguments to the analyze method,
+        such as `ad_hoc_recognizers`, `context`, `return_decision_process`.
+        See `AnalyzerEngine.analyze` for the full list.
+        """
+
+        context = []
+        if "context" in kwargs:
+            context = kwargs["context"]
+            del kwargs["context"]
+
+        if not keys_to_skip:
+            keys_to_skip = []
+
+        for key, value in input_dict.items():
+            if not value or key in keys_to_skip:
+                yield DictAnalyzerResult(key=key, value=value, recognizer_results=[])
+                continue  # skip this key as requested
+
+            # Add the key as an additional context
+            specific_context = context[:]
+            specific_context.append(key)
+
+            if type(value) in (str, int, bool, float):
+                results: List[RecognizerResult] = self.analyzer_engine.analyze(
+                    text=str(value), language=language, context=[key], **kwargs
+                )
+            elif isinstance(value, dict):
+                new_keys_to_skip = self._get_nested_keys_to_skip(key, keys_to_skip)
+                results = self.analyze_dict(
+                    input_dict=value,
+                    language=language,
+                    context=specific_context,
+                    keys_to_skip=new_keys_to_skip,
+                    **kwargs,
+                )
+            elif isinstance(value, Iterable):
+                # Recursively iterate nested dicts
+
+                results: List[List[RecognizerResult]] = self.analyze_iterator(
+                    texts=value,
+                    language=language,
+                    context=specific_context,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(f"type {type(value)} is unsupported.")
+
+            yield DictAnalyzerResult(key=key, value=value, recognizer_results=results)
+
+    @staticmethod
+    def _validate_types(value_iterator: Iterable[Any]) -> Iterator[Any]:
+        for val in value_iterator:
+            if val and not type(val) in (int, float, bool, str):
+                err_msg = (
+                    "Analyzer.analyze_iterator only works "
+                    "on primitive types (int, float, bool, str). "
+                    "Lists of objects are not yet supported."
+                )
+                logger.error(err_msg)
+                raise ValueError(err_msg)
+            yield val
+
+    @staticmethod
+    def _get_nested_keys_to_skip(key, keys_to_skip):
+        new_keys_to_skip = [
+            k.replace(f"{key}.", "") for k in keys_to_skip if k.startswith(key)
+        ]
+        return new_keys_to_skip

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/dict_analyzer_result.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from typing import List, Union, Iterator
+
+from presidio_analyzer import RecognizerResult
+
+
+@dataclass
+class DictAnalyzerResult:
+    """
+    Data class for holding the output of the Presidio Analyzer on dictionaries.
+
+    :param key: key in dictionary
+    :param value: value to run analysis on (either string or list of strings)
+    :param recognizer_results: Analyzer output for one value.
+    Could be either:
+     - A list of recognizer results if the input is one string
+     - A list of lists of recognizer results, if the input is a list of strings.
+     - An iterator of a DictAnalyzerResult, if the input is a dictionary.
+     In this case the recognizer_results would be the iterator
+     of the DictAnalyzerResults next level in the dictionary.
+    """
+
+    key: str
+    value: Union[str, List[str], dict]
+    recognizer_results: Union[
+        List[RecognizerResult],
+        List[List[RecognizerResult]],
+        Iterator["DictAnalyzerResult"],
+    ]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/entity_recognizer.py

@@ -0,0 +1,199 @@
+import logging
+from abc import abstractmethod
+from typing import List, Dict, Optional
+
+from presidio_analyzer import RecognizerResult
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class EntityRecognizer:
+    """
+    A class representing an abstract PII entity recognizer.
+
+    EntityRecognizer is an abstract class to be inherited by
+    Recognizers which hold the logic for recognizing specific PII entities.
+
+    EntityRecognizer exposes a method called enhance_using_context which
+    can be overridden in case a custom context aware enhancement is needed
+    in derived class of a recognizer.
+
+    :param supported_entities: the entities supported by this recognizer
+    (for example, phone number, address, etc.)
+    :param supported_language: the language supported by this recognizer.
+    The supported langauge code is iso6391Name
+    :param name: the name of this recognizer (optional)
+    :param version: the recognizer current version
+    :param context: a list of words which can help boost confidence score
+    when they appear in context of the matched entity
+    """
+
+    MIN_SCORE = 0
+    MAX_SCORE = 1.0
+
+    def __init__(
+        self,
+        supported_entities: List[str],
+        name: str = None,
+        supported_language: str = "en",
+        version: str = "0.0.1",
+        context: Optional[List[str]] = None,
+    ):
+
+        self.supported_entities = supported_entities
+
+        if name is None:
+            self.name = self.__class__.__name__  # assign class name as name
+        else:
+            self.name = name
+
+        self._id = f"{self.name}_{id(self)}"
+
+        self.supported_language = supported_language
+        self.version = version
+        self.is_loaded = False
+        self.context = context if context else []
+
+        self.load()
+        logger.info("Loaded recognizer: %s", self.name)
+        self.is_loaded = True
+
+    @property
+    def id(self):
+        """Return a unique identifier of this recognizer."""
+
+        return self._id
+
+    @abstractmethod
+    def load(self) -> None:
+        """
+        Initialize the recognizer assets if needed.
+
+        (e.g. machine learning models)
+        """
+
+    @abstractmethod
+    def analyze(
+        self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts
+    ) -> List[RecognizerResult]:
+        """
+        Analyze text to identify entities.
+
+        :param text: The text to be analyzed
+        :param entities: The list of entities this recognizer is able to detect
+        :param nlp_artifacts: A group of attributes which are the result of
+        an NLP process over the input text.
+        :return: List of results detected by this recognizer.
+        """
+        return None
+
+    def enhance_using_context(
+        self,
+        text: str,
+        raw_recognizer_results: List[RecognizerResult],
+        other_raw_recognizer_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """Enhance confidence score using context of the entity.
+
+        Override this method in derived class in case a custom logic
+        is needed, otherwise return value will be equal to
+        raw_results.
+
+        in case a result score is boosted, derived class need to update
+        result.recognition_metadata[RecognizerResult.IS_SCORE_ENHANCED_BY_CONTEXT_KEY]
+
+        :param text: The actual text that was analyzed
+        :param raw_recognizer_results: This recognizer's results, to be updated
+        based on recognizer specific context.
+        :param other_raw_recognizer_results: Other recognizer results matched in
+        the given text to allow related entity context enhancement
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param context: list of context words
+        """
+        return raw_recognizer_results
+
+    def get_supported_entities(self) -> List[str]:
+        """
+        Return the list of entities this recognizer can identify.
+
+        :return: A list of the supported entities by this recognizer
+        """
+        return self.supported_entities
+
+    def get_supported_language(self) -> str:
+        """
+        Return the language this recognizer can support.
+
+        :return: A list of the supported language by this recognizer
+        """
+        return self.supported_language
+
+    def get_version(self) -> str:
+        """
+        Return the version of this recognizer.
+
+        :return: The current version of this recognizer
+        """
+        return self.version
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {
+            "supported_entities": self.supported_entities,
+            "supported_language": self.supported_language,
+            "name": self.name,
+            "version": self.version,
+        }
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, entity_recognizer_dict: Dict) -> "EntityRecognizer":
+        """
+        Create EntityRecognizer from a dict input.
+
+        :param entity_recognizer_dict: Dict containing keys and values for instantiation
+        """
+        return cls(**entity_recognizer_dict)
+
+    @staticmethod
+    def remove_duplicates(results: List[RecognizerResult]) -> List[RecognizerResult]:
+        """
+        Remove duplicate results.
+
+        Remove duplicates in case the two results
+        have identical start and ends and types.
+        :param results: List[RecognizerResult]
+        :return: List[RecognizerResult]
+        """
+        results = list(set(results))
+        results = sorted(results, key=lambda x: (-x.score, x.start, -(x.end - x.start)))
+        filtered_results = []
+
+        for result in results:
+            if result.score == 0:
+                continue
+
+            to_keep = result not in filtered_results  # equals based comparison
+            if to_keep:
+                for filtered in filtered_results:
+                    # If result is contained in one of the other results
+                    if (
+                        result.contained_in(filtered)
+                        and result.entity_type == filtered.entity_type
+                    ):
+                        to_keep = False
+                        break
+
+            if to_keep:
+                filtered_results.append(result)
+
+        return filtered_results

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/local_recognizer.py

@@ -0,0 +1,7 @@
+from abc import ABC
+
+from presidio_analyzer import EntityRecognizer
+
+
+class LocalRecognizer(ABC, EntityRecognizer):
+    """PII entity recognizer which runs on the same process as the AnalyzerEngine."""

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/__init__.py

@@ -0,0 +1,19 @@
+"""NLP engine package. Performs text pre-processing."""
+
+from .nlp_artifacts import NlpArtifacts
+from .nlp_engine import NlpEngine
+from .spacy_nlp_engine import SpacyNlpEngine
+from .client_nlp_engine import ClientNlpEngine
+from .stanza_nlp_engine import StanzaNlpEngine
+from .transformers_nlp_engine import TransformersNlpEngine
+from .nlp_engine_provider import NlpEngineProvider
+
+__all__ = [
+    "NlpArtifacts",
+    "NlpEngine",
+    "SpacyNlpEngine",
+    "ClientNlpEngine",
+    "StanzaNlpEngine",
+    "NlpEngineProvider",
+    "TransformersNlpEngine",
+]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/client_nlp_engine.py

@@ -0,0 +1,108 @@
+import logging
+# import logging
+
+try:
+    import client
+    import spacy_client
+except ImportError:
+    client = None
+
+from typing import Optional, Dict, Iterator, Tuple, Union, List
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc
+
+from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class ClientNlpEngine(NlpEngine):
+    """
+    SpacyNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The SpacyNlpEngine uses SpaCy as its NLP module
+    """
+
+    
+    engine_name="spacy"
+    
+    is_available = bool(spacy)
+   
+
+    def __init__(self, models: Optional[Dict[str, str]] = None):
+        """
+        Initialize a wrapper on spaCy functionality.
+
+        :param models: Dictionary with the name of the spaCy model per language.
+        For example: models = {"en": "en_core_web_lg"}
+        """
+        if not models:
+            models = {"en": "en_core_web_lg"}
+        logger.debug(f"Loading SpaCy models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy.load(model_name, disable=["parser"])
+            for lang_code, model_name in models.items()
+        }
+
+        
+
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the SpaCy NLP pipeline on the given text and language."""
+
+        doc = self.nlp[language](text)
+        return self._doc_to_nlp_artifact(doc, language)
+
+    def process_batch(
+        self,
+        texts: Union[List[str], List[Tuple[str, object]]],
+        language: str,
+        as_tuples: bool = False,
+    ) -> Iterator[Optional[NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts using spacy pipe."""
+        texts = (str(text) for text in texts)
+        docs = self.nlp[language].pipe(texts, as_tuples=as_tuples)
+        for doc in docs:
+            yield doc.text, self._doc_to_nlp_artifact(doc, language)
+
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+        return self.nlp[language].vocab[word].is_stop
+
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language).
+        """
+        return self.nlp[language].vocab[word].is_punct
+
+    def get_nlp(self, language: str) -> Language:
+        """
+        Return the language model loaded for a language.
+
+        :param language: Name of language
+        :return: Language model from spaCy
+        """
+        return self.nlp[language]
+
+    def _doc_to_nlp_artifact(self, doc: Doc, language: str) -> NlpArtifacts:
+        lemmas = [token.lemma_ for token in doc]
+        tokens_indices = [token.idx for token in doc]
+        entities = doc.ents
+        return NlpArtifacts(
+            entities=entities,
+            tokens=doc,
+            tokens_indices=tokens_indices,
+            lemmas=lemmas,
+            nlp_engine=self,
+            language=language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py

@@ -0,0 +1,74 @@
+import json
+from typing import List
+
+from spacy.tokens import Doc, Span
+
+
+class NlpArtifacts:
+    """
+    NlpArtifacts is an abstraction layer over the results of an NLP pipeline.
+
+    processing over a given text, it holds attributes such as entities,
+    tokens and lemmas which can be used by any recognizer
+    """
+
+    def __init__(
+        self,
+        entities: List[Span],
+        tokens: Doc,
+        tokens_indices: List[int],
+        lemmas: List[str],
+        nlp_engine,  # noqa ANN001
+        language: str,
+    ):
+        self.entities = entities
+        self.tokens = tokens
+        self.lemmas = lemmas
+        self.tokens_indices = tokens_indices
+        self.keywords = self.set_keywords(nlp_engine, lemmas, language)
+        self.nlp_engine = nlp_engine
+
+    @staticmethod
+    def set_keywords(
+        nlp_engine, lemmas: List[str], language: str  # noqa ANN001
+    ) -> List[str]:
+        """
+        Return keywords fpr text.
+
+        Extracts lemmas with certain conditions as keywords.
+        """
+        if not nlp_engine:
+            return []
+        keywords = [
+            k.lower()
+            for k in lemmas
+            if not nlp_engine.is_stopword(k, language)
+            and not nlp_engine.is_punct(k, language)
+            and k != "-PRON-"
+            and k != "be"
+        ]
+
+        # best effort, try even further to break tokens into sub tokens,
+        # this can result in reducing false negatives
+        keywords = [i.split(":") for i in keywords]
+
+        # splitting the list can, if happened, will result in list of lists,
+        # we flatten the list
+        keywords = [item for sublist in keywords for item in sublist]
+        return keywords
+
+    def to_json(self) -> str:
+        """Convert nlp artifacts to json."""
+
+        return_dict = self.__dict__.copy()
+
+        # Ignore NLP engine as it's not serializable currently
+        del return_dict["nlp_engine"]
+
+        # Converting spaCy tokens and spans to string as they are not serializable
+        if "tokens" in return_dict:
+            return_dict["tokens"] = [token.text for token in self.tokens]
+        if "entities" in return_dict:
+            return_dict["entities"] = [entity.text for entity in self.entities]
+
+        return json.dumps(return_dict)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine.py

@@ -0,0 +1,42 @@
+from abc import ABC, abstractmethod
+from typing import Iterable, Iterator, Tuple
+
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+
+class NlpEngine(ABC):
+    """
+    NlpEngine is an abstraction layer over the nlp module.
+
+    It provides NLP preprocessing functionality as well as other queries
+    on tokens.
+    """
+
+    @abstractmethod
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the NLP pipeline on the given text and language."""
+
+    @abstractmethod
+    def process_batch(
+        self, texts: Iterable[str], language: str, **kwargs
+    ) -> Iterator[Tuple[str, NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts.
+
+        Returns a tuple of (text, NlpArtifacts)
+        """
+
+    @abstractmethod
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+
+    @abstractmethod
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language)
+        """

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/spacy_nlp_engine.py

@@ -0,0 +1,96 @@
+import logging
+from typing import Optional, Dict, Iterator, Tuple, Union, List
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc
+
+from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class SpacyNlpEngine(NlpEngine):
+    """
+    SpacyNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The SpacyNlpEngine uses SpaCy as its NLP module
+    """
+
+    engine_name = "spacy"
+    is_available = bool(spacy)
+   
+
+    def __init__(self, models: Optional[Dict[str, str]] = None):
+        """
+        Initialize a wrapper on spaCy functionality.
+
+        :param models: Dictionary with the name of the spaCy model per language.
+        For example: models = {"en": "en_core_web_lg"}
+        """
+        if not models:
+            models = {"en": "en_core_web_lg"}
+        logger.debug(f"Loading SpaCy models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy.load(model_name, disable=["parser"])
+            for lang_code, model_name in models.items()
+        }
+
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the SpaCy NLP pipeline on the given text and language."""
+
+        doc = self.nlp[language](text)
+        return self._doc_to_nlp_artifact(doc, language)
+
+    def process_batch(
+        self,
+        texts: Union[List[str], List[Tuple[str, object]]],
+        language: str,
+        as_tuples: bool = False,
+    ) -> Iterator[Optional[NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts using spacy pipe."""
+        texts = (str(text) for text in texts)
+        docs = self.nlp[language].pipe(texts, as_tuples=as_tuples)
+        for doc in docs:
+            yield doc.text, self._doc_to_nlp_artifact(doc, language)
+
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+        return self.nlp[language].vocab[word].is_stop
+
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language).
+        """
+        return self.nlp[language].vocab[word].is_punct
+
+    def get_nlp(self, language: str) -> Language:
+        """
+        Return the language model loaded for a language.
+
+        :param language: Name of language
+        :return: Language model from spaCy
+        """
+        return self.nlp[language]
+
+    def _doc_to_nlp_artifact(self, doc: Doc, language: str) -> NlpArtifacts:
+        lemmas = [token.lemma_ for token in doc]
+        tokens_indices = [token.idx for token in doc]
+        entities = doc.ents
+        return NlpArtifacts(
+            entities=entities,
+            tokens=doc,
+            tokens_indices=tokens_indices,
+            lemmas=lemmas,
+            nlp_engine=self,
+            language=language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py

@@ -0,0 +1,39 @@
+import logging
+
+try:
+    import stanza
+    import spacy_stanza
+except ImportError:
+    stanza = None
+
+from presidio_analyzer.nlp_engine import SpacyNlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class StanzaNlpEngine(SpacyNlpEngine):
+    """
+    StanzaNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The StanzaNlpEngine uses spacy-stanza and stanza as its NLP module
+
+    :param models: Dictionary with the name of the stanza model per language.
+    For example: models = {"en": "en"}
+    """
+
+    engine_name = "stanza"
+    is_available = bool(stanza)
+    def __init__(self, models=None):  # noqa ANN201
+        if not models:
+            models = {"en": "en"}
+        logger.debug(f"Loading Stanza models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy_stanza.load_pipeline(
+                model_name,
+                processors="tokenize,pos,lemma,ner",
+            )
+            for lang_code, model_name in models.items()
+        }

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/pattern.py

@@ -0,0 +1,45 @@
+import json
+from typing import Dict
+
+
+class Pattern:
+    """
+    A class that represents a regex pattern.
+
+    :param name: the name of the pattern
+    :param regex: the regex pattern to detect
+    :param score: the pattern's strength (values varies 0-1)
+    """
+
+    def __init__(self, name: str, regex: str, score: float):
+
+        self.name = name
+        self.regex = regex
+        self.score = score
+
+    def to_dict(self) -> Dict:
+        """
+        Turn this instance into a dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {"name": self.name, "score": self.score, "regex": self.regex}
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, pattern_dict: Dict) -> "Pattern":
+        """
+        Load an instance from a dictionary.
+
+        :param pattern_dict: a dictionary holding the pattern's parameters
+        :return: a Pattern instance
+        """
+        return cls(**pattern_dict)
+
+    def __repr__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())
+
+    def __str__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/pattern_recognizer.py

@@ -0,0 +1,253 @@
+import datetime
+import logging
+from typing import List, Optional, Dict
+
+import regex as re
+
+from presidio_analyzer import (
+    LocalRecognizer,
+    Pattern,
+    RecognizerResult,
+    EntityRecognizer,
+    AnalysisExplanation,
+)
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class PatternRecognizer(LocalRecognizer):
+    """
+    PII entity recognizer using regular expressions or deny-lists.
+
+    :param patterns: A list of patterns to detect
+    :param deny_list: A list of words to detect,
+    in case our recognizer uses a predefined list of words (deny list)
+    :param context: list of context words
+    :param deny_list_score: confidence score for a term
+    identified using a deny-list
+    """
+
+    def __init__(
+        self,
+        supported_entity: str,
+        name: str = None,
+        supported_language: str = "en",
+        patterns: List[Pattern] = None,
+        deny_list: List[str] = None,
+        context: List[str] = None,
+        deny_list_score: float = 1.0,
+        version: str = "0.0.1",
+    ):
+
+        if not supported_entity:
+            raise ValueError("Pattern recognizer should be initialized with entity")
+
+        if not patterns and not deny_list:
+            raise ValueError(
+                "Pattern recognizer should be initialized with patterns"
+                " or with deny list"
+            )
+
+        super().__init__(
+            supported_entities=[supported_entity],
+            supported_language=supported_language,
+            name=name,
+            version=version,
+        )
+        if patterns is None:
+            self.patterns = []
+        else:
+            self.patterns = patterns
+        self.context = context
+        self.deny_list_score = deny_list_score
+
+        if deny_list:
+            deny_list_pattern = self._deny_list_to_regex(deny_list)
+            self.patterns.append(deny_list_pattern)
+            self.deny_list = deny_list
+        else:
+            self.deny_list = []
+
+    def load(self):  # noqa D102
+        pass
+
+    def analyze(
+        self,
+        text: str,
+        entities: List[str],
+        nlp_artifacts: NlpArtifacts = None,
+        regex_flags: int = None,
+    ) -> List[RecognizerResult]:
+        """
+        Analyzes text to detect PII using regular expressions or deny-lists.
+
+        :param text: Text to be analyzed
+        :param entities: Entities this recognizer can detect
+        :param nlp_artifacts: Output values from the NLP engine
+        :param regex_flags:
+        :return:
+        """
+        results = []
+
+        if self.patterns:
+            pattern_result = self.__analyze_patterns(text, regex_flags)
+            results.extend(pattern_result)
+
+        return results
+
+    def _deny_list_to_regex(self, deny_list: List[str]) -> Pattern:
+        """
+        Convert a list of words to a matching regex.
+
+        To be analyzed by the analyze method as any other regex patterns.
+
+        :param deny_list: the list of words to detect
+        :return:the regex of the words for detection
+        """
+
+        # Escape deny list elements as preparation for regex
+        escaped_deny_list = [re.escape(element) for element in deny_list]
+        regex = r"(?:^|(?<=\W))(" + "|".join(escaped_deny_list) + r")(?:(?=\W)|$)"
+        return Pattern(name="deny_list", regex=regex, score=self.deny_list_score)
+
+    def validate_result(self, pattern_text: str) -> Optional[bool]:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        return None
+
+    def invalidate_result(self, pattern_text: str) -> Optional[bool]:
+        """
+        Logic to check for result invalidation by running pruning logic.
+
+        For example, each SSN number group should not consist of all the same digits.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the result is invalidated
+        """
+        return None
+
+    @staticmethod
+    def build_regex_explanation(
+        recognizer_name: str,
+        pattern_name: str,
+        pattern: str,
+        original_score: float,
+        validation_result: bool,
+    ) -> AnalysisExplanation:
+        """
+        Construct an explanation for why this entity was detected.
+
+        :param recognizer_name: Name of recognizer detecting the entity
+        :param pattern_name: Regex pattern name which detected the entity
+        :param pattern: Regex pattern logic
+        :param original_score: Score given by the recognizer
+        :param validation_result: Whether validation was used and its result
+        :return: Analysis explanation
+        """
+        explanation = AnalysisExplanation(
+            recognizer=recognizer_name,
+            original_score=original_score,
+            pattern_name=pattern_name,
+            pattern=pattern,
+            validation_result=validation_result,
+        )
+        return explanation
+
+    def __analyze_patterns(
+        self, text: str, flags: int = None
+    ) -> List[RecognizerResult]:
+        """
+        Evaluate all patterns in the provided text.
+
+        Including words in the provided deny-list
+
+        :param text: text to analyze
+        :param flags: regex flags
+        :return: A list of RecognizerResult
+        """
+        flags = flags if flags else re.DOTALL | re.MULTILINE
+        results = []
+        for pattern in self.patterns:
+            match_start_time = datetime.datetime.now()
+            matches = re.finditer(pattern.regex, text, flags=flags)
+            match_time = datetime.datetime.now() - match_start_time
+            logger.debug(
+                "--- match_time[%s]: %s.%s seconds",
+                pattern.name,
+                match_time.seconds,
+                match_time.microseconds,
+            )
+
+            for match in matches:
+                start, end = match.span()
+                current_match = text[start:end]
+
+                # Skip empty results
+                if current_match == "":
+                    continue
+
+                score = pattern.score
+
+                validation_result = self.validate_result(current_match)
+                description = self.build_regex_explanation(
+                    self.name, pattern.name, pattern.regex, score, validation_result
+                )
+                pattern_result = RecognizerResult(
+                    entity_type=self.supported_entities[0],
+                    start=start,
+                    end=end,
+                    score=score,
+                    analysis_explanation=description,
+                    recognition_metadata={
+                        RecognizerResult.RECOGNIZER_NAME_KEY: self.name,
+                        RecognizerResult.RECOGNIZER_IDENTIFIER_KEY: self.id,
+                    },
+                )
+
+                if validation_result is not None:
+                    if validation_result:
+                        pattern_result.score = EntityRecognizer.MAX_SCORE
+                    else:
+                        pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                invalidation_result = self.invalidate_result(current_match)
+                if invalidation_result is not None and invalidation_result:
+                    pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                if pattern_result.score > EntityRecognizer.MIN_SCORE:
+                    results.append(pattern_result)
+
+                # Update analysis explanation score following validation or invalidation
+                description.score = pattern_result.score
+
+        results = EntityRecognizer.remove_duplicates(results)
+        return results
+
+    def to_dict(self) -> Dict:
+        """Serialize instance into a dictionary."""
+        return_dict = super().to_dict()
+
+        return_dict["patterns"] = [pat.to_dict() for pat in self.patterns]
+        return_dict["deny_list"] = self.deny_list
+        return_dict["context"] = self.context
+        return_dict["supported_entity"] = return_dict["supported_entities"][0]
+        del return_dict["supported_entities"]
+
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, entity_recognizer_dict: Dict) -> "PatternRecognizer":
+        """Create instance from a serialized dict."""
+        patterns = entity_recognizer_dict.get("patterns")
+        if patterns:
+            patterns_list = [Pattern.from_dict(pat) for pat in patterns]
+            entity_recognizer_dict["patterns"] = patterns_list
+
+        return cls(**entity_recognizer_dict)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/recognizer_registry/__init__.py

@@ -0,0 +1,4 @@
+"""Recognizer registry init."""
+from .recognizer_registry import RecognizerRegistry
+
+__all__ = ["RecognizerRegistry"]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/recognizer_result.py

@@ -0,0 +1,189 @@
+import logging
+from typing import Dict
+
+from presidio_analyzer import AnalysisExplanation
+
+
+class RecognizerResult:
+    """
+    Recognizer Result represents the findings of the detected entity.
+
+    Result of a recognizer analyzing the text.
+
+    :param entity_type: the type of the entity
+    :param start: the start location of the detected entity
+    :param end: the end location of the detected entity
+    :param score: the score of the detection
+    :param analysis_explanation: contains the explanation of why this
+                                 entity was identified
+    :param recognition_metadata: a dictionary of metadata to be used in
+    recognizer specific cases, for example specific recognized context words
+    and recognizer name
+    """
+
+    # Keys for recognizer metadata
+    RECOGNIZER_NAME_KEY = "recognizer_name"
+    RECOGNIZER_IDENTIFIER_KEY = "recognizer_identifier"
+
+    # Key of a flag inside recognition_metadata dictionary
+    # which is set to true if the result enhanced by context
+    IS_SCORE_ENHANCED_BY_CONTEXT_KEY = "is_score_enhanced_by_context"
+
+    logger = logging.getLogger("presidio_analyzer")
+
+    def __init__(
+        self,
+        entity_type: str,
+        start: int,
+        end: int,
+        score: float,
+        analysis_explanation: AnalysisExplanation = None,
+        recognition_metadata: Dict = None,
+    ):
+
+        self.entity_type = entity_type
+        self.start = start
+        self.end = end
+        self.score = score
+        self.analysis_explanation = analysis_explanation
+
+        if not recognition_metadata:
+            self.logger.debug(
+                "recognition_metadata should be passed, "
+                "containing a recognizer_name value"
+            )
+
+        self.recognition_metadata = recognition_metadata
+
+    def append_analysis_explanation_text(self, text: str) -> None:
+        """Add text to the analysis explanation."""
+        if self.analysis_explanation:
+            self.analysis_explanation.append_textual_explanation_line(text)
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return self.__dict__
+
+    @classmethod
+    def from_json(cls, data: Dict) -> "RecognizerResult":
+        """
+        Create RecognizerResult from json.
+
+        :param data: e.g. {
+            "start": 24,
+            "end": 32,
+            "score": 0.8,
+            "entity_type": "NAME"
+        }
+        :return: RecognizerResult
+        """
+        score = data.get("score")
+        entity_type = data.get("entity_type")
+        start = data.get("start")
+        end = data.get("end")
+        return cls(entity_type, start, end, score)
+
+    def __repr__(self) -> str:
+        """Return a string representation of the instance."""
+        return self.__str__()
+
+    def intersects(self, other: "RecognizerResult") -> int:
+        """
+        Check if self intersects with a different RecognizerResult.
+
+        :return: If intersecting, returns the number of
+        intersecting characters.
+        If not, returns 0
+        """
+        # if they do not overlap the intersection is 0
+        if self.end < other.start or other.end < self.start:
+            return 0
+
+        # otherwise the intersection is min(end) - max(start)
+        return min(self.end, other.end) - max(self.start, other.start)
+
+    def contained_in(self, other: "RecognizerResult") -> bool:
+        """
+        Check if self is contained in a different RecognizerResult.
+
+        :return: true if contained
+        """
+        return self.start >= other.start and self.end <= other.end
+
+    def contains(self, other: "RecognizerResult") -> bool:
+        """
+        Check if one result is contained or equal to another result.
+
+        :param other: another RecognizerResult
+        :return: bool
+        """
+        return self.start <= other.start and self.end >= other.end
+
+    def equal_indices(self, other: "RecognizerResult") -> bool:
+        """
+        Check if the indices are equal between two results.
+
+        :param other: another RecognizerResult
+        :return:
+        """
+        return self.start == other.start and self.end == other.end
+
+    def __gt__(self, other: "RecognizerResult") -> bool:
+        """
+        Check if one result is greater by using the results indices in the text.
+
+        :param other: another RecognizerResult
+        :return: bool
+        """
+        if self.start == other.start:
+            return self.end > other.end
+        return self.start > other.start
+
+    def __eq__(self, other: "RecognizerResult") -> bool:
+        """
+        Check two results are equal by using all class fields.
+
+        :param other: another RecognizerResult
+        :return: bool
+        """
+        equal_type = self.entity_type == other.entity_type
+        equal_score = self.score == other.score
+        return self.equal_indices(other) and equal_type and equal_score
+
+    def __hash__(self):
+        """
+        Hash the result data by using all class fields.
+
+        :return: int
+        """
+        return hash(
+            f"{str(self.start)} {str(self.end)} {str(self.score)} {self.entity_type}"
+        )
+
+    def __str__(self) -> str:
+        """Return a string representation of the instance."""
+        return (
+            f"type: {self.entity_type}, "
+            f"start: {self.start}, "
+            f"end: {self.end}, "
+            f"score: {self.score}"
+        )
+
+    def has_conflict(self, other: "RecognizerResult") -> bool:
+        """
+        Check if two recognizer results are conflicted or not.
+
+        I have a conflict if:
+        1. My indices are the same as the other and my score is lower.
+        2. If my indices are contained in another.
+
+        :param other: RecognizerResult
+        :return:
+        """
+        if self.equal_indices(other):
+            return self.score <= other.score
+        return other.contains(self)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/remote_recognizer.py

@@ -0,0 +1,57 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from presidio_analyzer import EntityRecognizer
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+
+class RemoteRecognizer(ABC, EntityRecognizer):
+    """
+    A configuration for a recognizer that runs on a different process / remote machine.
+
+    :param supported_entities: A list of entities this recognizer can identify
+    :param name: name of recognizer
+    :param supported_language: The language this recognizer can detect entities in
+    :param version: Version of this recognizer
+    """
+
+    def __init__(
+        self,
+        supported_entities: List[str],
+        name: Optional[str],
+        supported_language: str,
+        version: str,
+        context: Optional[List[str]] = None,
+    ):
+        super().__init__(
+            supported_entities=supported_entities,
+            name=name,
+            supported_language=supported_language,
+            version=version,
+            context=context,
+        )
+
+    @abstractmethod
+    def load(self):  # noqa D102
+        pass
+
+    @abstractmethod
+    def analyze(
+        self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts
+    ):  # noqa ANN201
+        """
+        Call an external service for PII detection.
+
+        :param text: text to be analyzed
+        :param entities: Entities that should be looked for
+        :param nlp_artifacts: Additional metadata from the NLP engine
+        :return: List of identified PII entities
+        """
+
+        # 1. Call the external service.
+        # 2. Translate results into List[RecognizerResult]
+        pass
+
+    @abstractmethod
+    def get_supported_entities(self) -> List[str]:  # noqa D102
+        pass

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/setup.py

@@ -0,0 +1 @@
+import setuptools
                 name='presidio_analyzer',
                 version='4.1.0',
                 author='Amit Hegde',
                 author_email='amitumamaheshwar.h@infosys.com',
                 description='Infosys Intelligent Assistant',
                 long_description='Infosys Intelligent Assistant',
                 classifiers=['Programming Language :: Python :: 3', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent'],
                 package_dir={'': 'presidio_analyzer'},
                 packages=setuptools.find_packages(where='presidio_analyzer'),
                 python_requires='>=3.6',
                 )

presidio_analyzer/presidio_analyzer/Package_to_wheel.txt

@@ -0,0 +1,5 @@
+1.Make necessary changes in build_config file Like Package name and version
+
+2.pip install pyc_wheel build
+
+3.python create_wheel_file.py --> Creates Wheel file