# -*- coding: utf-8 -*-
"""
Created on Tue Jun 14 23:53:47 2022

@author: UTKARSH
"""

import numpy as np

import re
import glob
import spacy
from sklearn.feature_extraction.text import TfidfVectorizer

try:
    from src.clean import clean_license_text
except:
    from clean import clean_license_text


nlp = spacy.load("en_core_web_sm")

TOP_N_WORDS = 100


def tfidf_preprocess(text):
    """
    Cleans text by lowercasing it and removing all the special characters.

    Parameters
    ----------
    text : str
        Raw license text.

    Returns
    -------
    str
        Cleaned and lowercased license text.

    """
    text = text.lower()

    # Remove all non-letter words
    text = re.sub("[^a-zA-Z]+", " ", text)

    ## Lemmatize the words
    # text = " ".join([token.lemma_.lower().strip() for token in nlp(text)])
    return text.strip()

corpus = list()

filepaths = glob.glob("../notebooks/licenses/NOASSERTION/*.txt")
# filepaths.extend(glob.glob("../notebooks/licenses/OTHER/*.txt"))

for file_path in filepaths:
    with open(file_path, "r", encoding="utf-8") as f:
        # To eliminate the url and blank line from start of the files
        f.readline()
        f.readline()

        # Reading the remaining content
        content = f.read()

        cleaned_license_text, _ = clean_license_text(content)
        corpus.append(cleaned_license_text)


vectorizer = TfidfVectorizer(
    lowercase=True,
    preprocessor=tfidf_preprocess,
    stop_words="english"
)

tfidf = vectorizer.fit_transform(corpus)

feature_array = np.array(vectorizer.get_feature_names_out())
tfidf_sorting = np.argsort(tfidf.toarray()).flatten()

top_n = feature_array[tfidf_sorting][-TOP_N_WORDS:][::-1]
bottom_n = feature_array[tfidf_sorting][:TOP_N_WORDS]