# Import necessary libraries
import nltk
import spacy
from keybert import KeyBERT
from nltk.stem import WordNetLemmatizer
from spellchecker import SpellChecker
import yake
import re
from rake_nltk import Rake
from collections import Counter

# Ensure required NLTK resources are downloaded
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('omw-1.4')

# Load spaCy's transformer-based English model
# Note: This model requires installation: python -m spacy download en_core_web_trf
spacy_nlp = spacy.load("en_core_web_trf")  # Transformer-based model

# Initialize models and tools
spell = SpellChecker()
lemmatizer = WordNetLemmatizer()
kw_model = KeyBERT(model='all-mpnet-base-v2')  # Enhanced embeddings
rake_extractor = Rake(
    stopwords=nltk.corpus.stopwords.words('english'),
    max_length=3
)

# Custom stop words list
custom_stop_words = set(nltk.corpus.stopwords.words('english'))
custom_stop_words.update([
    # Include domain-specific stop words and fillers
    'charlie', 'papa', 'number', 'sitenumber', 'thank', 'thanks', 'please',
    'hello', 'hi', 'bye', 'good', 'great', 'ok', 'okay', 'yes', 'yeah',
    'leon', 'vivian', 'victor', 'nana', 'um', 'uh', 'like', 'just', 'really', 'know', 'get',
    'got', 'make', 'go', 'going', 'us', 'would', 'could', 'should', 'well', 'right', 'am',
    'pm', 'gonna', 'wanna', 'youre', 'ive', 'dont', 'didnt', 'cant', 'wont', 'are',
    'is', 'was', 'were', 'all right', 'much', 'enjoy', 'day', 'today', 'also',
    'still', 'oh', 'one', 'two', 'three', 'five', 'zero', 'double', 'give', 'see', 'thank',
    'im', 'need', 'access', 'site', 'number', 'hours', 'hour', 'call', 'calling',
    'assist', 'help', 'sorry', 'thing', 'bye-bye', 'no',
    'alright', 'sure', 'great', 'like', 'thanks', 'thankyou',
    # Add any other fillers or irrelevant words
])

# Function to preprocess text
def preprocess_text(text):
    # Lowercase text
    text = text.lower()

    # Correct common mishearings or transcription errors
    text = text.replace('side', 'site')

    # Expand contractions
    contractions = {
        "can't": "cannot",
        "won't": "will not",
        "don't": "do not",
        "i'm": "i am",
        "you're": "you are",
        # Add more contractions as needed
    }
    for contraction, expanded in contractions.items():
        text = text.replace(contraction, expanded)

    # Replace number words with digits
    num_dict = {
        'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4',
        'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9',
        'oh': '0', 'double': '', 'triple': '', 'hundred': '00', 'thousand': '000'
    }
    for word, digit in num_dict.items():
        text = re.sub(r'\b{}\b'.format(word), digit, text)

    # Merge separated digits into numbers
    text = re.sub(r'(\d+)\s+(\d+)', r'\1\2', text)
    text = re.sub(r'\s+', ' ', text)

    # Merge multi-word entities
    text = re.sub(r'charlie\s*papa\s*(\d+)', r'charliepapa\1', text)
    text = re.sub(r'site\s*number\s*(\d+)', r'sitenumber\1', text)

    # Remove punctuation
    text = re.sub(r'[^\w\s]', '', text)

    # Tokenize
    tokens = nltk.word_tokenize(text)

    # Correct spelling errors and handle None values
    tokens = [spell.correction(word) or word for word in tokens]

    # Remove any None or empty strings
    tokens = [word for word in tokens if word]

    # POS Tagging
    pos_tags = nltk.pos_tag(tokens)

    # Keep only nouns and proper nouns
    tokens = [word for word, pos in pos_tags if pos.startswith('NN')]

    # Lemmatize
    tokens = [lemmatizer.lemmatize(word) for word in tokens]

    # Remove stop words
    tokens = [word for word in tokens if word not in custom_stop_words]

    return ' '.join(tokens)

# Function to extract keywords
def extract_keywords(sentence, num_keywords=10):
    # Preprocess text
    clean_text = preprocess_text(sentence)

    # Process original sentence with spaCy for NER
    doc = spacy_nlp(sentence)

    # Extract named entities using NER
    ents = [ent.text for ent in doc.ents if ent.label_ in {
        'PERSON', 'ORG', 'GPE', 'LOC', 'FAC', 'PRODUCT', 'EVENT', 'WORK_OF_ART',
        'LAW', 'LANGUAGE', 'DATE', 'TIME', 'MONEY', 'PERCENT', 'QUANTITY',
        'ORDINAL', 'CARDINAL'
    }]

    # Keyword extraction with KeyBERT
    keybert_keywords = kw_model.extract_keywords(
        clean_text,
        keyphrase_ngram_range=(1, 3),
        stop_words=list(custom_stop_words),
        top_n=num_keywords * 2,
        use_mmr=True,
        diversity=0.8
    )
    keybert_keywords = [kw[0] for kw in keybert_keywords]

    # Keyword extraction with YAKE
    yake_extractor = yake.KeywordExtractor(
        lan='en',
        n=3,
        dedupLim=0.8,
        top=num_keywords,
        features=None
    )
    yake_keywords = yake_extractor.extract_keywords(clean_text)
    yake_keywords = [kw[0] for kw in yake_keywords]

    # Keyword extraction with RAKE
    rake_extractor.extract_keywords_from_text(clean_text)
    rake_keywords = rake_extractor.get_ranked_phrases()[:num_keywords]

    # Combine keywords and remove duplicates
    combined_keywords = set(ents + keybert_keywords + yake_keywords + rake_keywords)

    # Remove stop words and short words
    final_keywords = [
        kw for kw in combined_keywords
        if kw.lower() not in custom_stop_words and len(kw) > 2
    ]

    # Rank keywords by frequency in the cleaned text
    keyword_frequency = Counter()
    for kw in final_keywords:
        keyword_frequency[kw] = clean_text.count(kw)

    # Sort keywords by frequency
    sorted_keywords = [kw for kw, freq in keyword_frequency.most_common(num_keywords)]

    return sorted_keywords

# Your sentences
sentences = [
    "Thank you for calling me to see now for speaking. Can I assist you? I need access to a site, please. Glam key number? Charlie Papa 87. Charlie Papa 87. 87. Speaking to Leon. Leon, yeah. Site number? 353380. 353380. Site number three five double three eight zero three five double three eight zero, yeah, I'm going to the main listening. No, I'm saying how long? How long? Give us two hours, please. All right, you cannot make sure to close any of that. I'll do that. Thank you. All right, thank you. Bye. Bye-bye.",
    "Thanks for calling me. It was nice. Yeah. That's nice, yeah you need only mangate and the DC tank are all the logs including the DP and all the new sample. What are you going to do inside? What are you going to do inside? What are you going to do inside? Oh, okay. There. Yes. Yes. Yes. Yes. Yes do you be inside? Maybe two hours. Yes, it is being granted with me. You recognize it? No, thank you. Thank you so much. I love you. Okay, you too. Thank you. Bye.",
    "Thank you for calling. It is you through to Vivian. May I assist you? Hi Vivian, how are you? I'm good, thanks. How are you? I'm okay. Can you please help me with access please, Nana? Certainly. Can I have a site number, Victor? Site number, Victor? Hello? Yes, it's Victor from World Construction. Can I have a site number? Site number? 352612. Is it a secondary? Yes, it's my number. You're from rural we are all construction every ID it's A2-0 and 0 so the teacher in class how my country outside to come give us five hours access granted is anything I can accept it I think I appreciate you thank you so much enjoy your day you too well bye"
]

# Process each sentence and extract keywords
for idx, sentence in enumerate(sentences, 1):
    print(f"Sentence {idx}:\n{sentence}\n")
    # Extract keywords
    keywords = extract_keywords(sentence)
    print("Extracted Keywords:")
    for kw in keywords:
        print("-", kw)
    print("-" * 80)



---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[6], line 3
      1 # Import necessary libraries
      2 import nltk
----> 3 import spacy
      4 from keybert import KeyBERT
      5 from nltk.stem import WordNetLemmatizer

File ~\AppData\Local\anaconda3\Lib\site-packages\spacy\__init__.py:6
      3 from typing import Any, Dict, Iterable, Union
      5 # set library-specific custom warning handling before doing anything else
----> 6 from .errors import setup_default_warnings
      8 setup_default_warnings()  # noqa: E402
     10 # These are imported as part of the API

File ~\AppData\Local\anaconda3\Lib\site-packages\spacy\errors.py:3
      1 import warnings
----> 3 from .compat import Literal
      6 class ErrorsWithCodes(type):
      7     def __getattribute__(self, code):

File ~\AppData\Local\anaconda3\Lib\site-packages\spacy\compat.py:4
      1 """Helpers for Python and platform compatibility."""
      2 import sys
----> 4 from thinc.util import copy_array
      6 try:
      7     import cPickle as pickle

File ~\AppData\Local\anaconda3\Lib\site-packages\thinc\__init__.py:5
      2 import numpy
      4 from .about import __version__
----> 5 from .config import registry
      7 # fmt: off
      8 __all__ = [
      9     "registry",
     10     "__version__",
     11 ]

File ~\AppData\Local\anaconda3\Lib\site-packages\thinc\config.py:5
      2 import confection
      3 from confection import VARIABLE_RE, Config, ConfigValidationError, Promise
----> 5 from .types import Decorator
      8 class registry(confection.registry):
      9     # fmt: off
     10     optimizers: Decorator = catalogue.create("thinc", "optimizers", entry_points=True)

File ~\AppData\Local\anaconda3\Lib\site-packages\thinc\types.py:25
      4 from typing import (
      5     Any,
      6     Callable,
   (...)
     20     overload,
     21 )
     23 import numpy
---> 25 from .compat import cupy, has_cupy
     27 if has_cupy:
     28     get_array_module = cupy.get_array_module

File ~\AppData\Local\anaconda3\Lib\site-packages\thinc\compat.py:99
     95 has_mxnet = False
     98 try:
---> 99     import h5py
    100 except ImportError:  # pragma: no cover
    101     h5py = None

File ~\AppData\Local\anaconda3\Lib\site-packages\h5py\__init__.py:45
     36     _warn(("h5py is running against HDF5 {0} when it was built against {1}, "
     37            "this may cause problems").format(
     38             '{0}.{1}.{2}'.format(*version.hdf5_version_tuple),
     39             '{0}.{1}.{2}'.format(*version.hdf5_built_version_tuple)
     40     ))
     43 _errors.silence_errors()
---> 45 from ._conv import register_converters as _register_converters, \
     46                    unregister_converters as _unregister_converters
     47 _register_converters()
     48 atexit.register(_unregister_converters)

File h5py\\_conv.pyx:1, in init h5py._conv()

File h5py\\h5r.pyx:1, in init h5py.h5r()

File h5py\\h5p.pyx:1, in init h5py.h5p()

ValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject