init commit

app.py

@@ -0,0 +1,24 @@
+import gradio as gr
+from text_complexity_analyzer_cm.perm import PERM
+
+
+def predict(text):
+    perm = PERM('en')
+    analytics = []
+    analytics.append(perm.calculate_word_information_indices_for_one_text(text, workers=-1))
+    analytics.append(perm.calculate_descriptive_indices_for_one_text(text, workers=-1))
+    return analytics
+
+
+title = "Get the Analytics of your Message"
+
+iface = gr.Interface(fn=predict,
+                     inputs=gr.inputs.Textbox(
+                         lines=3, label='Insert any given text to get textual analytics.'),
+                     outputs="text",
+                     title=title,
+                     theme="huggingface",
+                     examples=[
+                         'We are going to analyze this text for persuasiveness, word information, and descriptive information.']
+                     )
+iface.launch()

requirements.txt

@@ -0,0 +1,3 @@
+spacy
+Pyphen
+python3 -m spacy download en_core_web_md

text_complexity_analyzer_cm/__init__.py

@@ -0,0 +1,3 @@
+'''
+This module contains the classes and functions that handle the processing of text using the coh-metrix indices.
+'''

text_complexity_analyzer_cm/coh_metrix_indices/__init__.py

@@ -0,0 +1,3 @@
+'''
+This module contains the classes to calculate the coh metrix indices.
+'''

text_complexity_analyzer_cm/coh_metrix_indices/connective_indices.py

@@ -0,0 +1,77 @@
+import multiprocessing
+import pyphen
+import spacy
+import string
+
+from typing import Callable
+from typing import List
+from text_complexity_analyzer_cm.coh_metrix_indices.descriptive_indices import DescriptiveIndices
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+from text_complexity_analyzer_cm.utils.utils import split_text_into_sentences
+
+class ConnectiveIndices:
+    def __init__(self, nlp, language: str='en', descriptive_indices: DescriptiveIndices=None) -> None:
+        self.language = language
+        self._nlp = nlp
+        self._incidence = 1
+        if descriptive_indices is None: 
+            self._di = DescriptiveIndices(language)
+        else:
+            self._di = descriptive_indices
+
+    def _get_connectives_incidence(self, text: str, disable_pipeline: List, count_connectives_function: Callable, word_count: int=None, workers: int=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text) 
+        pc = len(paragraphs)
+        threads = 1
+        wc = word_count if word_count is not None else self._di.get_word_count_from_text(text)
+        self._nlp.get_pipe('feature counter').counter_function = count_connectives_function
+        connectives = sum(doc._.feature_count for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads))
+        return connectives
+
+    def get_causal_connectives_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['causal connective tagger', 'feature counter']]
+        causal_connectives_counter = lambda doc: len(doc._.causal_connectives)
+        result = self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=causal_connectives_counter, workers=workers)
+        return result
+
+    def get_temporal_connectives_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['temporal connective tagger', 'feature counter']]
+        temporal_connectives_counter = lambda doc: len(doc._.temporal_connectives)
+        result = self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=temporal_connectives_counter, workers=workers)
+        return result
+
+    def get_exemplifications_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['exemplifications tagger', 'tagger', 'feature counter']]
+        exemplifications_counter = lambda doc: len(doc._.exemplifications)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=exemplifications_counter, workers=workers)
+
+    def get_emphatics_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['emphatics tagger', 'tagger', 'feature counter']]
+        emphatics_counter = lambda doc: len(doc._.emphatics)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=emphatics_counter, workers=workers)
+
+    def get_asks_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['asks tagger', 'tagger', 'feature counter']]
+        asks_counter = lambda doc: len(doc._.asks)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=asks_counter, workers=workers)
+
+    def get_polites_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['polites tagger', 'tagger', 'feature counter']]
+        polites_counter = lambda doc: len(doc._.polites)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=polites_counter, workers=workers)
+
+    def get_logical_connectives_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['logical connective tagger', 'tagger', 'feature counter']]
+        logical_connectives_counter = lambda doc: len(doc._.logical_connectives)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=logical_connectives_counter, workers=workers)
+
+    def get_adversative_connectives_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['adversative connective tagger', 'tagger', 'feature counter']]
+        adversative_connectives_counter = lambda doc: len(doc._.adversative_connectives)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=adversative_connectives_counter, workers=workers)
+
+    def get_additive_connectives_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['additive connective tagger', 'tagger', 'feature counter']]
+        additive_connectives_counter = lambda doc: len(doc._.additive_connectives)
+        return self._get_connectives_incidence(text, disable_pipeline=disable_pipeline, count_connectives_function=additive_connectives_counter, workers=workers)

text_complexity_analyzer_cm/coh_metrix_indices/descriptive_indices.py

@@ -0,0 +1,123 @@
+import multiprocessing
+import spacy
+import statistics
+import string
+
+from typing import Callable
+from typing import List
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES, LANGUAGES_DICTIONARY_PYPHEN
+from text_complexity_analyzer_cm.pipes.syllable_splitter import SyllableSplitter
+from text_complexity_analyzer_cm.utils.statistics_results import StatisticsResults
+from text_complexity_analyzer_cm.utils.utils import is_word
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+from text_complexity_analyzer_cm.utils.utils import split_text_into_sentences
+from text_complexity_analyzer_cm.utils.utils import split_doc_into_sentences
+
+
+class DescriptiveIndices:
+    def __init__(self, nlp, language: str='en') -> None:
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+        
+        self.language = language
+        self._nlp = nlp
+
+    def get_paragraph_count_from_text(self, text: str) -> int:
+        if len(text) == 0:
+            raise ValueError('The text is empty.')
+        
+        return len(split_text_into_paragraphs(text))
+
+    def get_sentence_count_from_text(self, text: str, workers: int=-1) -> int:
+        if len(text) == 0:
+            raise ValueError('The text is empty.')
+        elif workers == 0 or workers < -1:
+            raise ValueError('Workers must be -1 or any positive number greater than 0')
+        else:
+            paragraphs = split_text_into_paragraphs(text) # Obtain paragraphs
+            threads = 1#multiprocessing.cpu_count() if workers == -1 else workers  
+            disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['sentencizer', 'feature counter']]
+
+            sentence_counter = lambda doc: sum(1 for _ in doc.sents)
+            self._nlp.get_pipe('feature counter').counter_function = sentence_counter
+            
+            sentences = sum(doc._.feature_count
+                            for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads))
+            return sentences
+
+    def get_word_count_from_text(self, text: str, workers: int=-1) -> int:
+        if len(text) == 0:
+            raise ValueError('The text is empty.')
+        elif workers == 0 or workers < -1:
+            raise ValueError('Workers must be -1 or any positive number greater than 0')
+        else:
+            paragraphs = split_text_into_paragraphs(text) # Obtain paragraphs
+            threads = 1#multiprocessing.cpu_count() if workers == -1 else workers
+            word_counter = lambda doc: sum(1 for token in doc if is_word(token))
+            disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe != 'feature counter']
+            self._nlp.get_pipe('feature counter').counter_function = word_counter
+
+            total_words = sum(doc._.feature_count for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads))
+
+
+            return total_words
+
+    def _get_mean_std_of_metric(self, text: str, disable_pipeline: List, counter_function: Callable, statistic_type: str='all', workers=-1) -> StatisticsResults:
+        paragraphs = split_text_into_paragraphs(text)
+        threads = 1
+        self._nlp.get_pipe('feature counter').counter_function = counter_function
+        counter = []
+
+        for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads):
+            current_result = doc._.feature_count # Find the values to add to the counter
+            if not isinstance(current_result, list): # Add any numbers
+                counter.append(current_result)
+            else:
+                if len(current_result) > 0: # Only add values if its not an empty array
+                    counter.extend(current_result)
+
+        stat_results = StatisticsResults()
+        if statistic_type in ['std', 'all']:
+            stat_results.std = statistics.pstdev(counter)
+        
+        if statistic_type in ['mean', 'all']:
+            stat_results.mean = statistics.mean(counter)
+
+        return stat_results
+
+    def get_length_of_paragraphs(self, text: str, workers: int=-1) -> StatisticsResults:
+        count_length_of_paragraphs = lambda doc: sum(1 for _ in split_doc_into_sentences(doc))
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['sentencizer', 'feature counter']]
+        return self._get_mean_std_of_metric(text, disable_pipeline=disable_pipeline, counter_function=count_length_of_paragraphs, statistic_type='all', workers=workers)
+
+    def get_length_of_sentences(self, text: str, workers: int=-1) -> StatisticsResults:
+        count_length_of_sentences = lambda doc: [len([1 for token in sentence
+                                                    if is_word(token)])
+                                                    for sentence in doc.sents]
+
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['sentencizer', 'feature counter']]
+
+        return self._get_mean_std_of_metric(text, disable_pipeline=disable_pipeline, counter_function=count_length_of_sentences, statistic_type='all', workers=workers)
+
+    def get_length_of_words(self, text: str, workers: int=-1) -> StatisticsResults:
+
+        count_letters_per_word = lambda doc: [len(token)
+                                            for token in doc
+                                            if is_word(token)]
+        
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe != 'feature counter']
+        
+        result = self._get_mean_std_of_metric(text, disable_pipeline=disable_pipeline, counter_function=count_letters_per_word, statistic_type='all', workers=workers)
+        
+        return result
+
+    def get_syllables_per_word(self, text: str, workers=-1) -> StatisticsResults:
+        count_syllables_per_word = lambda doc: [len(token._.syllables)
+                                                for token in doc
+                                                if is_word(token) and token._.syllables is not None]
+        
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['syllables', 'feature counter']]
+        
+        result = self._get_mean_std_of_metric(text, disable_pipeline=disable_pipeline, counter_function=count_syllables_per_word, statistic_type='all', workers=workers)
+    
+        return result 

text_complexity_analyzer_cm/coh_metrix_indices/lexical_diversity_indices.py

@@ -0,0 +1,37 @@
+import multiprocessing
+import spacy
+import string
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.utils import is_content_word
+from text_complexity_analyzer_cm.utils.utils import is_word
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+
+class LexicalDiversityIndices:
+    def __init__(self, nlp, language: str='en') -> None:
+        self.language = language
+        self._nlp = nlp
+
+    def get_type_token_ratio_between_all_words(self, text: str, workers=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text)
+        threads = 1
+        tokens = []
+        disable_pipeline = []
+
+        tokens = [token.text.lower()
+                for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads)
+                for token in doc
+                if is_word(token)]
+
+        return 0 if len(tokens) == 0 else len(set(tokens)) / len(tokens)
+
+    def get_type_token_ratio_of_content_words(self, text: str, workers=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text)
+        threads = 1
+        tokens = []
+        disable_pipeline = [] 
+        tokens = [token.text.lower()
+                for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads)
+                for token in doc
+                if is_content_word(token)]
+        return 0 if len(tokens) == 0 else len(set(tokens)) / len(tokens)

text_complexity_analyzer_cm/coh_metrix_indices/readability_indices.py

@@ -0,0 +1,66 @@
+import multiprocessing
+
+import spacy
+
+from text_complexity_analyzer_cm.coh_metrix_indices.descriptive_indices import DescriptiveIndices
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+
+class ReadabilityIndices:
+    '''
+    This class will handle all operations to find the readability indices of a text according to Coh-Metrix.
+    '''
+
+    def __init__(self, nlp, language: str='en', descriptive_indices: DescriptiveIndices=None) -> None:
+        '''
+        The constructor will initialize this object that calculates the readability indices for a specific language of those that are available.
+
+        Parameters:
+        nlp: The spacy model that corresponds to a language.
+        language(str): The language that the texts to process will have.
+        descriptive_indices(DescriptiveIndices): The class that calculates the descriptive indices of a text in a certain language.
+
+        Returns:
+        None.
+        '''
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+        elif descriptive_indices is not None and descriptive_indices.language != language:
+            raise ValueError(f'The descriptive indices analyzer must be of the same language as the word information analyzer.')
+        
+        self.language = language
+        self._nlp = nlp
+
+        if descriptive_indices is None: # Assign the descriptive indices to an attribute
+            self._di = DescriptiveIndices(language=language, nlp=nlp)
+        else:
+            self._di = descriptive_indices
+
+    def calculate_fernandez_huertas_grade_level(self, text: str=None, mean_syllables_per_word: int=None, mean_words_per_sentence: int=None, workers: int=-1) -> float:
+        '''
+        This function obtains the Fernández-Huertas readability index for a text.
+
+        Parameters:
+        text(str): The text to be analized.
+        word_count(int): The amount of words in the text.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+        mean_syllables_per_word(int): The mean of syllables per word in the text.
+        mean_words_per_sentence(int): The mean amount of words per sentences in the text.
+
+        Returns:
+        float: The Fernández-Huertas readability index for a text.
+        '''
+        if self.language != 'en':
+            raise ValueError('This readability index is for spanish.')
+        elif text is not None and len(text) == 0:
+            raise ValueError('The word is empty.')
+        elif text is None and (mean_syllables_per_word is None or mean_words_per_sentence is None):
+            raise ValueError('If there\'s no text, then you must pass mean_syllables_per_word and mean_words_per_sentence at the same time.')
+        elif workers == 0 or workers < -1:
+            raise ValueError('Workers must be -1 or any positive number greater than 0')
+        else:
+            threads = multiprocessing.cpu_count() if workers == -1 else workers
+            mspw = mean_syllables_per_word if mean_syllables_per_word is not None else self._di.get_mean_of_syllables_per_word(text=text, workers=threads)
+            mwps = mean_words_per_sentence if mean_words_per_sentence is not None else self._di.get_mean_of_length_of_sentences(text=text, workers=threads)
+            
+            return 206.84 - 0.6 * mspw - 1.02 * mwps

text_complexity_analyzer_cm/coh_metrix_indices/referential_cohesion_indices.py

@@ -0,0 +1,402 @@
+import multiprocessing
+import spacy
+import statistics
+
+from itertools import combinations
+from spacy.tokens import Span
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.statistics_results import StatisticsResults
+from text_complexity_analyzer_cm.utils.utils import is_word
+from text_complexity_analyzer_cm.utils.utils import is_content_word
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+from typing import Callable
+from typing import List
+
+
+class ReferentialCohesionIndices:
+    '''
+    This class will handle all operations to find the synthactic pattern density indices of a text according to Coh-Metrix.
+    '''
+    # TODO: Implement multiprocessing
+    def __init__(self, nlp, language: str='en') -> None:
+        '''
+        The constructor will initialize this object that calculates the synthactic pattern density indices for a specific language of those that are available.
+
+        Parameters:
+        nlp: The spacy model that corresponds to a language.
+        language(str): The language that the texts to process will have.
+
+        Returns:
+        None.
+        '''
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+
+        self.language = language
+        self._nlp = nlp
+
+    def _calculate_overlap_for_adjacent_sentences(self, text: str, disable_pipeline: List, sentence_analyzer: Callable, statistic_type: str='mean', workers: int=-1) -> StatisticsResults:
+        '''
+        This method calculates the overlap for adjacent sentences in a text. MULTIPROCESSING STILL NOT IMPLEMENTED.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        disable_pipeline(List): The pipeline elements to be disabled.
+        sentence_analyzer(Callable): The function that analyzes sentences to check cohesion.
+        statistic_type(str): Whether to calculate the mean and/or the standard deviation. It accepts 'mean', 'std' or 'all'.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        StatisticsResults: The standard deviation and mean of the overlap.
+        '''
+        # TODO MULTIPROCESSING. WORKERS IS JUST A PLACEHOLDER
+        if len(text) == 0:
+            raise ValueError('The text is empty.')
+        elif statistic_type not in ['mean', 'std', 'all']:
+            raise ValueError('\'statistic_type\' can only take \'mean\', \'std\' or \'all\'.')
+        else:
+            self._nlp.get_pipe('referential cohesion adjacent sentences analyzer').sentence_analyzer = sentence_analyzer
+            doc = self._nlp(text, disable=disable_pipeline)
+            stat_results = StatisticsResults() # Create empty container
+
+            if len(doc._.referential_cohesion_adjacent) == 0:
+                return stat_results
+            else:
+                if statistic_type in ['mean', 'all']:
+                    stat_results.mean = statistics.mean(doc._.referential_cohesion_adjacent)
+
+                if statistic_type in ['std', 'all']:
+                    stat_results.std = statistics.pstdev(doc._.referential_cohesion_adjacent)
+                
+                return stat_results
+
+    def _calculate_overlap_for_all_sentences(self, text: str, disable_pipeline: List, sentence_analyzer: Callable, statistic_type: str='all', workers: int=-1) -> StatisticsResults:
+        '''
+        This method calculates the overlap for all sentences in a text. MULTIPROCESSING STILL NOT IMPLEMENTED.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        disable_pipeline(List): The pipeline elements to be disabled.
+        sentence_analyzer(Callable): The function that analyzes sentences to check cohesion.
+        statistic_type(str): Whether to calculate the mean and/or the standard deviation. It accepts 'mean', 'std' or 'all'.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        StatisticsResults: The standard deviation and mean of the overlap.
+        '''
+        # TODO MULTIPROCESSING. WORKERS IS JUST A PLACEHOLDER.
+        if len(text) == 0:
+            raise ValueError('The text is empty.')
+        elif statistic_type not in ['mean', 'std', 'all']:
+            raise ValueError('\'statistic_type\' can only take \'mean\', \'std\' or \'all\'.')
+        else:
+            self._nlp.get_pipe('referential cohesion all sentences analyzer').sentence_analyzer = sentence_analyzer
+            doc = self._nlp(text, disable=disable_pipeline)
+            stat_results = StatisticsResults() # Create empty container
+
+            if len(doc._.referential_cohesion_all) == 0:
+                return stat_results
+            else:
+                if statistic_type in ['mean', 'all']:
+                    stat_results.mean = statistics.mean(doc._.referential_cohesion_all)
+
+                if statistic_type in ['std', 'all']:
+                    stat_results.std = statistics.pstdev(doc._.referential_cohesion_all)
+                
+                return stat_results
+
+    def get_noun_overlap_adjacent_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the noun overlap for adjacent sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean noun overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion adjacent sentences analyzer']]
+        return self._calculate_overlap_for_adjacent_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_noun_overlap, statistic_type='mean').mean
+
+    def get_noun_overlap_all_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the noun overlap for all sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean noun overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion all sentences analyzer']]
+        return self._calculate_overlap_for_all_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_noun_overlap, statistic_type='mean').mean
+
+    def get_argument_overlap_adjacent_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the argument overlap for adjacent sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean argument overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion adjacent sentences analyzer']]
+        return self._calculate_overlap_for_adjacent_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_argument_overlap, statistic_type='mean').mean
+
+    def get_argument_overlap_all_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the argument overlap for all sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean argument overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion all sentences analyzer']]
+        return self._calculate_overlap_for_all_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_argument_overlap, statistic_type='mean').mean
+
+    def get_stem_overlap_adjacent_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the stem overlap for adjacent sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean stem overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion adjacent sentences analyzer']]
+        return self._calculate_overlap_for_adjacent_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_stem_overlap, statistic_type='mean').mean
+
+    def get_stem_overlap_all_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the stem overlap for all sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean stem overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion all sentences analyzer']]
+        return self._calculate_overlap_for_all_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_stem_overlap, statistic_type='mean').mean
+
+    def get_content_word_overlap_adjacent_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the mean and standard deviation of the content word overlap for adjacent sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean mean and standard deviation of the content word overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion adjacent sentences analyzer']]
+        return self._calculate_overlap_for_adjacent_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_content_word_overlap, statistic_type='all')
+
+    def get_content_word_overlap_all_sentences(self, text: str, workers: int=-1) -> StatisticsResults:
+        '''
+        This method calculates the mean and standard deviation of the content word overlap for all sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        StatisticsResults: The mean mean and standard deviation of the content word overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion all sentences analyzer']]
+        return self._calculate_overlap_for_all_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_content_word_overlap, statistic_type='all')
+
+    def get_anaphore_overlap_adjacent_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the mean of the anaphore overlap for adjacent sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean mean of the anaphore overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion adjacent sentences analyzer']]
+        return self._calculate_overlap_for_adjacent_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_anaphore_overlap, statistic_type='all').mean
+
+    def get_anaphore_overlap_all_sentences(self, text: str, workers: int=-1) -> float:
+        '''
+        This method calculates the mean of the anaphore overlap for all sentences in a text.
+
+        Parameters:
+        text(str): The text to be analyzed.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        float: The mean mean of the anaphore overlap.
+        '''
+        disable_pipeline = [pipe
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['sentencizer', 'tagger', 'referential cohesion all sentences analyzer']]
+        return self._calculate_overlap_for_all_sentences(text=text, workers=workers, disable_pipeline=disable_pipeline, sentence_analyzer=analyze_anaphore_overlap, statistic_type='all').mean
+
+def analyze_noun_overlap(prev_sentence: Span, cur_sentence: Span, language: str='en') -> int:
+    '''
+    This function analyzes whether or not there's noun overlap between two sentences for a language.
+
+    Parameters:
+    prev_sentence(Span): The previous sentence to analyze.
+    cur_sentence(Span): The current sentence to analyze.
+    language(str): The language of the sentences.
+
+    Returns:
+    int: 1 if there's overlap between the two sentences and 0 if no.
+    '''
+    # Place the tokens in a dictionary for search efficiency
+    prev_sentence_noun_tokens = {token.text.lower(): None
+                                 for token in prev_sentence
+                                 if is_word(token) and token.pos_ == 'NOUN'}
+
+    for token in cur_sentence:
+        if language == 'en':
+            if is_word(token) and token.pos_ == 'NOUN' and token.text.lower() in prev_sentence_noun_tokens:
+                return 1 # There's cohesion
+
+    return 0 # No cohesion
+
+
+def analyze_argument_overlap(prev_sentence: Span, cur_sentence: Span, language: str='en') -> int:
+    '''
+    This function analyzes whether or not there's argument overlap between two sentences.
+
+    Parameters:
+    prev_sentence(Span): The previous sentence to analyze.
+    cur_sentence(Span): The current sentence to analyze.
+    language(str): The language of the sentences.
+
+    Returns:
+    int: 1 if there's overlap between the two sentences and 0 if no.
+    '''
+    # Place the tokens in a dictionary for search efficiency
+    prev_sentence_noun_tokens = {token.lemma_.lower(): None
+                                 for token in prev_sentence
+                                 if is_word(token) and token.pos_ == 'NOUN'}
+
+    prev_sentence_personal_pronouns_tokens = {token.text.lower(): None
+                                              for token in prev_sentence
+                                              if is_word(token) and 'PronType=Prs' in token.tag_}
+
+    for token in cur_sentence: # Iterate every token of the current sentence
+        if language == 'en':
+            if is_word(token) and token.pos_ == 'NOUN' and token.lemma_.lower() in prev_sentence_noun_tokens:
+                return 1 # There's cohesion by noun lemma
+
+            if is_word(token) and 'PronType=Prs' in token.tag_ and token.text.lower() in prev_sentence_personal_pronouns_tokens:
+                return 1 # There's cohesion by personal pronoun
+
+    return 0 # No cohesion
+
+
+def analyze_stem_overlap(prev_sentence: Span, cur_sentence: Span, language: str='en') -> int:
+    '''
+    This function analyzes whether or not there's stem overlap between two sentences.
+
+    Parameters:
+    prev_sentence(Span): The previous sentence to analyze.
+    cur_sentence(Span): The current sentence to analyze.
+    language(str): The language of the sentences.
+
+    Returns:
+    int: 1 if there's overlap between the two sentences and 0 if no.
+    '''
+    # Place the tokens in a dictionary for search efficiency
+    prev_sentence_content_stem_tokens = {token.lemma_.lower(): None
+                                         for token in prev_sentence
+                                         if is_content_word(token)}
+
+    for token in cur_sentence:
+        if language == 'en':
+            if is_word(token) and token.pos_ in ['NOUN', 'PROPN'] and token.lemma_.lower() in prev_sentence_content_stem_tokens:
+                return 1 # There's cohesion
+
+    return 0 # No cohesion
+
+
+def analyze_content_word_overlap(prev_sentence: Span, cur_sentence: Span, language='en') -> float:
+    '''
+    This function calculates the proportional content word overlap between two sentences.
+
+    Parameters:
+    prev_sentence(Span): The previous sentence to analyze.
+    cur_sentence(Span): The current sentence to analyze.
+    language(str): The language of the sentences.
+
+    Returns:
+    float: Proportion of tokens that overlap between the current and previous sentences
+    '''
+    total_tokens = len([token for token in prev_sentence if is_content_word(token)]) + len([token for token in cur_sentence if is_content_word(token)])
+
+    if total_tokens == 0: # Nothing to compute
+        return 0
+    else:
+        prev_sentence_content_words_tokens = {token.text.lower(): None
+                                              for token in prev_sentence
+                                              if is_content_word(token)}
+        matches = 0 # Matcher counter
+
+        for token in cur_sentence:
+            if language == 'en':
+                if is_content_word(token) and token.text.lower() in prev_sentence_content_words_tokens:
+                    matches += 2 # There's cohesion
+
+        return matches / total_tokens
+
+
+def analyze_anaphore_overlap(prev_sentence: Span, cur_sentence: Span, language: str='en') -> int:
+    '''
+    This function analyzes whether or not there's anaphore overlap between two sentences.
+
+    Parameters:
+    prev_sentence(Span): The previous sentence to analyze.
+    cur_sentence(Span): The current sentence to analyze.
+    language(str): The language of the sentences.
+
+    Returns:
+    int: 1 if there's overlap between the two sentences and 0 if no.
+    '''
+    # Place the tokens in a dictionary for search efficiency
+    prev_sentence_pronoun_tokens = {token.text.lower(): None
+                                    for token in prev_sentence
+                                    if is_word(token) and token.pos_ == 'PRON'}
+
+    for token in cur_sentence:
+        if language == 'en':
+            if is_word(token) and token.pos_ == 'PRON' and token.text.lower() in prev_sentence_pronoun_tokens:
+                return 1 # There's cohesion
+
+    return 0 # No cohesion

text_complexity_analyzer_cm/coh_metrix_indices/syntactic_complexity_indices.py

@@ -0,0 +1,56 @@
+import multiprocessing
+from typing import Tuple
+
+import spacy
+import statistics
+
+from spacy.tokens import Span
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.utils import is_word
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+from text_complexity_analyzer_cm.utils.utils import split_doc_into_sentences
+
+
+class SyntacticComplexityIndices:
+    def __init__(self, nlp, language: str='en') -> None:
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+        
+        self.language = language
+        self._nlp = nlp
+
+    def get_mean_number_of_modifiers_per_noun_phrase(self, text: str, workers: int=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text)
+        threads = 1
+        modifiers_per_noun_phrase = []
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['parser', 'tagger', 'noun phrase tagger', 'feature counter']]
+        modifiers_counter = lambda doc: [sum(1 for token in nph if token.pos_ == 'ADJ')
+                                        for nph in doc._.noun_phrases]
+        self._nlp.get_pipe('feature counter').counter_function = modifiers_counter
+        modifiers_per_noun_phrase = []
+
+        for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads):
+            modifiers_per_noun_phrase.extend(doc._.feature_count)
+        
+        return statistics.mean(modifiers_per_noun_phrase)
+
+    def get_mean_number_of_words_before_main_verb(self, text: str, workers: int=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text)
+        threads = 1
+        words_before_main_verb = []
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['feature counter',  'sentencizer']]
+        words_before_main_verb_counter = lambda doc: [amount_of_words_before_main_verb(s) for s in split_doc_into_sentences(doc)]
+        self._nlp.get_pipe('feature counter').counter_function = words_before_main_verb_counter
+        for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads):
+            words_before_main_verb.extend(doc._.feature_count)
+        return statistics.mean(words_before_main_verb)
+
+def amount_of_words_before_main_verb(sentence: Span) -> int:
+    left_words = []
+    for token in sentence:
+        if token.pos_ in ['VERB', 'AUX'] and token.dep_ == 'ROOT':
+            break
+        else:
+            if is_word(token):
+                left_words.append(token.text)
+    return len(left_words)

text_complexity_analyzer_cm/coh_metrix_indices/syntactic_pattern_density_indices.py

@@ -0,0 +1,126 @@
+import multiprocessing
+
+import spacy
+
+from typing import Callable
+from typing import List
+from text_complexity_analyzer_cm.coh_metrix_indices.descriptive_indices import DescriptiveIndices
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+
+
+class SyntacticPatternDensityIndices:
+    '''
+    This class will handle all operations to find the synthactic pattern density indices of a text according to Coh-Metrix.
+    '''
+
+    def __init__(self, nlp, language: str='en', descriptive_indices: DescriptiveIndices=None) -> None:
+        '''
+        The constructor will initialize this object that calculates the synthactic pattern density indices for a specific language of those that are available.
+
+        Parameters:
+        nlp: The spacy model that corresponds to a language.
+        language(str): The language that the texts to process will have.
+        descriptive_indices(DescriptiveIndices): The class that calculates the descriptive indices of a text in a certain language.
+
+        Returns:
+        None.
+        '''
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+        elif descriptive_indices is not None and descriptive_indices.language != language:
+            raise ValueError(f'The descriptive indices analyzer must be of the same language as the word information analyzer.')
+        
+        self.language = language
+        self._nlp = nlp
+        self._incidence = 1000
+
+        if descriptive_indices is None: # Assign the descriptive indices to an attribute
+            self._di = DescriptiveIndices(language=language, nlp=nlp)
+        else:
+            self._di = descriptive_indices
+
+    def _get_syntactic_pattern_density(self, text: str, disable_pipeline: List, sp_counter_function: Callable=None, word_count: int=None, workers: int=-1) -> int:
+        '''
+        This function obtains the incidence of a syntactic pattern that exist on a text per {self._incidence} words.
+
+        Parameters:
+        text(str): The text to be analized.
+        disable_pipeline(List): The pipeline elements to be disabled.
+        sp_counter_function(Callable): The function that counts a syntactic pattern for a Spacy document. It returns an integer.
+        word_count(int): The amount of words in the text.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        int: The incidence of a syntactic pattern per {self._incidence} words.
+        '''
+        if len(text) == 0:
+            raise ValueError('The word is empty.')
+        elif workers == 0 or workers < -1:
+            raise ValueError('Workers must be -1 or any positive number greater than 0')
+        else:
+            paragraphs = split_text_into_paragraphs(text) # Find all paragraphs
+            threads = multiprocessing.cpu_count() if workers == -1 else workers
+            wc = word_count if word_count is not None else self._di.get_word_count_from_text(text)            
+            self._nlp.get_pipe('feature counter').counter_function = sp_counter_function
+            density = sum(doc._.feature_count
+                          for doc in self._nlp.pipe(paragraphs, batch_size=threads, disable=disable_pipeline, n_process=threads)) # Calculate with multiprocessing 
+            
+            return (density / wc) * self._incidence
+
+    def get_noun_phrase_density(self, text: str, word_count: int=None, workers: int=-1) -> int:
+        '''
+        This function obtains the incidence of noun phrases that exist on a text per {self._incidence} words.
+
+        Parameters:
+        text(str): The text to be analized.
+        word_count(int): The amount of words in the text.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        int: The incidence of noun phrases per {self._incidence} words.
+        '''
+        count_noun_phrases = lambda doc: len(doc._.noun_phrases)
+        disable_pipeline = [pipe 
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['noun phrase tagger', 'tagger', 'parser', 'feature counter']]
+
+        return self._get_syntactic_pattern_density(text, disable_pipeline=disable_pipeline, sp_counter_function=count_noun_phrases, workers=workers)
+
+    def get_verb_phrase_density(self, text: str, word_count: int=None, workers: int=-1) -> int:
+        '''
+        This function obtains the incidence of verb phrases that exist on a text per {self._incidence} words.
+
+        Parameters:
+        text(str): The text to be analized.
+        word_count(int): The amount of words in the text.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        int: The incidence of verb phrases per {self._incidence} words.
+        '''
+        count_verb_phrases = lambda doc: len(doc._.verb_phrases)
+        disable_pipeline = [pipe 
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['verb phrase tagger', 'tagger', 'feature counter']]
+
+        return self._get_syntactic_pattern_density(text, disable_pipeline=disable_pipeline, sp_counter_function=count_verb_phrases, workers=workers)
+            
+    def get_negation_expressions_density(self, text: str, word_count: int=None, workers: int=-1) -> int:
+        '''
+        This function obtains the incidence of negation expressions that exist on a text per {self._incidence} words.
+
+        Parameters:
+        text(str): The text to be analized.
+        word_count(int): The amount of words in the text.
+        workers(int): Amount of threads that will complete this operation. If it's -1 then all cpu cores will be used.
+
+        Returns:
+        int: The incidence of negation expressions per {self._incidence} words.
+        '''
+        count_negation_expressions = lambda doc: len(doc._.negation_expressions)
+        disable_pipeline = [pipe 
+                            for pipe in self._nlp.pipe_names
+                            if pipe not in ['negative expression tagger', 'tagger', 'feature counter']]
+
+        return self._get_syntactic_pattern_density(text, disable_pipeline=disable_pipeline, sp_counter_function=count_negation_expressions, workers=workers)

text_complexity_analyzer_cm/coh_metrix_indices/word_information_indices.py

@@ -0,0 +1,99 @@
+import multiprocessing
+import spacy
+
+from typing import Callable
+from typing import List
+from text_complexity_analyzer_cm.coh_metrix_indices.descriptive_indices import DescriptiveIndices
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.utils.utils import is_word
+from text_complexity_analyzer_cm.utils.utils import split_text_into_paragraphs
+
+class WordInformationIndices:
+    def __init__(self, nlp, language: str='en', descriptive_indices: DescriptiveIndices=None) -> None:
+        self.language = language
+        self._nlp = nlp
+        self._incidence = 1000
+        if descriptive_indices is None:
+            self._di = DescriptiveIndices(language=language, nlp=nlp)
+        else:
+            self._di = descriptive_indices
+
+    def _get_word_type_incidence(self, text: str, disable_pipeline :List, counter_function: Callable, word_count: int=None, workers: int=-1) -> float:
+        paragraphs = split_text_into_paragraphs(text)
+        wc = word_count if word_count is not None else self._di.get_word_count_from_text(text)
+        self._nlp.get_pipe('feature counter').counter_function = counter_function
+        words = sum(doc._.feature_count for doc in self._nlp.pipe(paragraphs, batch_size=1, disable=disable_pipeline, n_process=1))
+        result = words #(words / wc)
+        return result
+
+    def get_noun_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        noun_counter = lambda doc: sum(1 for token in doc if is_word(token) and token.pos_ in ['NOUN', 'PROPN'])
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        result = self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=noun_counter, workers=workers)
+        return result
+
+    def get_verb_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        verb_counter = lambda doc: sum(1 for token in doc if is_word(token) and token.pos_ == 'VERB')
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=verb_counter, workers=workers)
+
+    def get_adjective_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        adjective_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'ADJ')
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=adjective_counter, workers=workers)
+
+    def get_adverb_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        adverb_counter = lambda doc: sum(1
+                                        for token in doc
+                                        if is_word(token) and token.pos_ == 'ADV')
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=adverb_counter, workers=workers)
+
+    def get_personal_pronoun_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                        for token in doc
+                                        if is_word(token) and token.pos_ == 'PRON')
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_first_person_singular_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'PRON' and 'Number=Sing' in token.morph and 'Person=1' in token.morph)
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_first_person_plural_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'PRON' and 'Number=Plur' in token.morph and 'Person=1' in token.morph)
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_second_person_singular_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'PRON' and 'Number=Sing' in token.morph and 'Person=2' in token.morph)
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_second_person_plural_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'PRON' and 'Number=Plur' in token.morph and 'Person=2' in token.morph)
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_third_person_singular_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1
+                                            for token in doc
+                                            if is_word(token) and token.pos_ == 'PRON' and 'Number=Sing' in token.morph and 'Person=3' in token.morph)        
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)
+
+    def get_personal_pronoun_third_person_plural_form_incidence(self, text: str, word_count: int=None, workers: int=-1) -> float:
+        pronoun_counter = lambda doc: sum(1 for token in doc if is_word(token) and token.pos_ == 'PRON' and 'Number=Plur' in token.morph and 'Person=3' in token.morph)
+        disable_pipeline = [pipe for pipe in self._nlp.pipe_names if pipe not in ['tok2vec', 'tagger', 'attribute_ruler', 'feature counter']]
+        return self._get_word_type_incidence(text, disable_pipeline=disable_pipeline, counter_function=pronoun_counter, workers=workers)

text_complexity_analyzer_cm/constants.py

@@ -0,0 +1,22 @@
+'''
+This function contains constants that will be used across the entire library.
+'''
+
+import os
+
+language = {
+    'es': 'es_core_news_lg',
+    'en': 'en_core_web_sm'
+}
+
+ACCEPTED_LANGUAGES = {
+    'es': 'es_core_news_lg',
+    'en': 'en_core_web_sm',
+}
+
+LANGUAGES_DICTIONARY_PYPHEN = {
+    'es': 'es',
+    'en': 'en'
+}
+
+BASE_DIRECTORY = os.path.dirname(os.path.abspath(__file__))

text_complexity_analyzer_cm/perm.py

@@ -0,0 +1,79 @@
+import spacy
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+from text_complexity_analyzer_cm.coh_metrix_indices.connective_indices import ConnectiveIndices
+from text_complexity_analyzer_cm.coh_metrix_indices.descriptive_indices import DescriptiveIndices
+from text_complexity_analyzer_cm.coh_metrix_indices.lexical_diversity_indices import LexicalDiversityIndices
+from text_complexity_analyzer_cm.coh_metrix_indices.syntactic_complexity_indices import SyntacticComplexityIndices
+from text_complexity_analyzer_cm.coh_metrix_indices.word_information_indices import WordInformationIndices
+from text_complexity_analyzer_cm.pipes.syllable_splitter import SyllableSplitter
+from text_complexity_analyzer_cm.pipes.causal_connectives_tagger import CausalConnectivesTagger
+from text_complexity_analyzer_cm.pipes.emphatics_tagger import EmphaticsTagger
+from text_complexity_analyzer_cm.pipes.asks_tagger import AsksTagger
+from text_complexity_analyzer_cm.pipes.polites_tagger import PolitesTagger
+from text_complexity_analyzer_cm.pipes.logical_connectives_tagger import LogicalConnectivesTagger
+from text_complexity_analyzer_cm.pipes.adversative_connectives_tagger import AdversativeConnectivesTagger
+from text_complexity_analyzer_cm.pipes.temporal_connectives_tagger import TemporalConnectivesTagger
+from text_complexity_analyzer_cm.pipes.additive_connectives_tagger import AdditiveConnectivesTagger
+from text_complexity_analyzer_cm.pipes.feature_counter import FeatureCounter
+from typing import Dict
+
+class PERM:
+    def __init__(self, language:str = 'en') -> None:
+        self.language = language
+        self._nlp = spacy.load(ACCEPTED_LANGUAGES[language], disable=['ner'])
+        self._nlp.max_length = 3000000
+        self._nlp.add_pipe('sentencizer')
+        self._nlp.add_pipe('syllables', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('causal connective tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('temporal connective tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('emphatics tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('asks tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('polites tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('logical connective tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('adversative connective tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('additive connective tagger', config={"language": 'en'}, after='tagger')
+        self._nlp.add_pipe('feature counter', config={"language": 'en'}, last=True)
+        self._di = DescriptiveIndices(language=language, nlp=self._nlp)
+        self._ci = ConnectiveIndices(language=language, nlp=self._nlp, descriptive_indices=self._di)
+        self._ldi = LexicalDiversityIndices(language=language, nlp=self._nlp)
+        self._sci = SyntacticComplexityIndices(language=language, nlp=self._nlp)
+        self._wii = WordInformationIndices(language=language, nlp=self._nlp, descriptive_indices=self._di)
+
+    def calculate_descriptive_indices_for_one_text(self, text: str, workers: int=-1) -> Dict:
+        indices = {}
+
+        indices['Paragraph Count'] = self._di.get_paragraph_count_from_text(text=text)
+        indices['Sentence Count'] = self._di.get_sentence_count_from_text(text=text, workers=workers)
+        indices['Word Count'] = self._di.get_word_count_from_text(text=text, workers=workers)
+        length_of_paragraph = self._di.get_length_of_paragraphs(text=text, workers=workers)
+        indices['Mean Length of Paragraphs'] = length_of_paragraph.mean
+        length_of_sentences = self._di.get_length_of_sentences(text=text, workers=workers)
+        indices['Mean Length of Sentences'] = length_of_sentences.mean
+        length_of_words = self._di.get_length_of_words(text=text, workers=workers)
+        indices['Mean Length of Words'] = length_of_words.mean
+        syllables_per_word = self._di.get_syllables_per_word(text=text, workers=workers)
+        indices['Mean Syllables of Words'] = syllables_per_word.mean
+
+        return indices
+
+    def calculate_word_information_indices_for_one_text(self, text: str, workers: int=-1, word_count: int=None) -> Dict:
+        indices = {}
+        
+        indices['#Nouns'] = self._wii.get_noun_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Verbs'] = self._wii.get_verb_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Adjectives'] = self._wii.get_adjective_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Adverbs'] = self._wii.get_adverb_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Personal Pronouns'] = self._wii.get_personal_pronoun_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Pers1s'] = self._wii.get_personal_pronoun_first_person_singular_form_incidence(text=text, workers=workers, word_count=word_count)
+        indices['#Pers1p'] = self._wii.get_personal_pronoun_first_person_plural_form_incidence(
+            text=text, workers=workers, word_count=word_count)
+        indices['#Pers2s'] = self._wii.get_personal_pronoun_second_person_singular_form_incidence(
+            text=text, workers=workers, word_count=word_count)
+        indices['#Pers2p'] = self._wii.get_personal_pronoun_second_person_plural_form_incidence(
+            text=text, workers=workers, word_count=word_count)
+        indices['#Pers3s'] = self._wii.get_personal_pronoun_third_person_singular_form_incidence(
+            text=text, workers=workers, word_count=word_count)
+        indices['#Pers3p'] = self._wii.get_personal_pronoun_third_person_plural_form_incidence(
+            text=text, workers=workers, word_count=word_count)
+        
+        return indices

text_complexity_analyzer_cm/pipes/__init__.py

@@ -0,0 +1,3 @@
+'''
+This module contains the Spacy pipes used to calculate certain coh-metrix indices.
+'''

text_complexity_analyzer_cm/pipes/additive_connectives_tagger.py

@@ -0,0 +1,59 @@
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Doc
+from spacy.tokens import Span
+from spacy.util import filter_spans
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+additive_connectives_getter = lambda doc: [doc[span['start']:span['end']]
+                                           for span in doc._.additive_connectives_span_indices]
+
+Doc.set_extension('additive_connectives_span_indices', force=False, default=[])
+Doc.set_extension('additive_connectives', force=False, getter=additive_connectives_getter)
+
+@Language.factory('additive connective tagger')
+class AdditiveConnectivesTagger:
+    def __init__(self, name, nlp, language: str='en') -> None:
+        '''
+        This constructor will initialize the object that tags additive connectives.
+
+        Parameters:
+        nlp: The Spacy model to use this tagger with.
+        language: The language that this pipeline will be used in.
+
+        Returns:
+        None.
+        '''
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+
+        self._language = language
+        self._matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
+        self._connectives = []
+        if language == 'en': # Temporal connectives for spanish
+            self._connectives = ['then', 'moreover', 'after', 'from here on', 'even', 'next', 'to top it all', 'another', 'finally', 'of equal importance', 'is more', 'first', 'besides', 'gradually', 'too', 'last', 'equally important', 'third', 'as soon as', 'on the other hand', 'furthermore', 'to begin with', 'above', 'also', 'first ', 'likewise', 'in addition', 'second', 'inclusive', 'further', 'before', 'hence', 'in the end', 'last of all']
+        else: # Support for future languages
+            #     self._connectives = ['asimismo', 'igualmente' 'de igual modo', 'de igual manera', 'de igual forma', 'del mismo modo', 'de la misma manera', 'de la misma forma', 'en primer lugar', 'en segundo lugar', 'en tercer lugar', 'en último lugar', 'por su parte', 'por otro lado', 'además', 'encima', 'es más', 'por añadidura', 'incluso', 'inclusive', 'para colmo']
+            pass
+
+        for con in self._connectives:
+            self._matcher.add(con, None, nlp(con))
+        
+
+    def __call__(self, doc: Doc) -> Doc:
+        '''
+        This method will find all additive connectives and store them in an iterable.
+
+        Parameters:
+        doc(Doc): A Spacy document.
+        '''
+        matches = self._matcher(doc)
+        additive_connectives_spans = [doc[start:end] for _, start, end in matches]
+
+        doc._.additive_connectives_span_indices = [{'start': span.start,
+                                                    'end': span.end,
+                                                    'label': span.label}
+                                                    for span in filter_spans(additive_connectives_spans)] # Save the temporal connectives found
+            
+        return doc

text_complexity_analyzer_cm/pipes/adversative_connectives_tagger.py

@@ -0,0 +1,64 @@
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Doc
+from spacy.tokens import Span
+from spacy.util import filter_spans
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+adversative_connectives_getter = lambda doc: [doc[span['start']:span['end']]
+                                            for span in doc._.adversative_connectives_span_indices]
+
+Doc.set_extension('adversative_connectives_span_indices', force=False, default=[])
+Doc.set_extension('adversative_connectives', force=False, getter=adversative_connectives_getter)
+
+@Language.factory('adversative connective tagger')
+class AdversativeConnectivesTagger:
+    '''
+    This tagger has the task to find all adversative connectives in a document. It needs to go after the 'Tagger' pipeline component.
+    '''
+    name = 'adversative connective tagger'
+
+    def __init__(self, name, nlp, language: str='en') -> None:
+        '''
+        This constructor will initialize the object that tags adversative connectives.
+
+        Parameters:
+        nlp: The Spacy model to use this tagger with.
+        language: The language that this pipeline will be used in.
+
+        Returns:
+        None.
+        '''
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+
+        self._language = language
+        self._matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
+        self._connectives = []
+        if language == 'en': # Adversative connectives for spanish
+            self._connectives = ['although', 'instead', 'and yet', 'nonetheless', 'nevertheless', 'rather', 'more than', 'yet', 'in fact', 'on the other hand', 'but yes', 'now well', 'on the contrary', 'however', 'in spite of this', 'conversely', 'still', 'less', 'actually', 'but rather', 'in contrast', 'but', 'except', 'only']
+            # self._connectives = ['pero', 'sino', 'no obstante', 'sino que', 'sin embargo', 'pero sí', 'aunque', 'menos', 'solo', 'excepto', 'salvo', 'más que', 'en cambio', 'ahora bien', 'más bien']
+        else: # Support for future languages
+            pass
+
+        for con in self._connectives:
+            self._matcher.add(con, None, nlp(con))
+        
+
+    def __call__(self, doc: Doc) -> Doc:
+        '''
+        This method will find all adversative connectives and store them in an iterable.
+
+        Parameters:
+        doc(Doc): A Spacy document.
+        '''
+        matches = self._matcher(doc)
+        adversative_connectives_spans = [doc[start:end] for _, start, end in matches]
+
+        doc._.adversative_connectives_span_indices = [{'start': span.start,
+                                                       'end': span.end,
+                                                       'label': span.label}
+                                                      for span in filter_spans(adversative_connectives_spans)] # Save the causal connectives found
+        
+        return doc

text_complexity_analyzer_cm/pipes/asks_tagger.py

@@ -0,0 +1,37 @@
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Doc
+from spacy.tokens import Span
+from spacy.util import filter_spans
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+asks_getter = lambda doc: [doc[span['start']:span['end']] for span in doc._.asks_span_indices]
+
+Doc.set_extension('asks_span_indices', force=False, default=[])
+Doc.set_extension('asks', force=False, getter=asks_getter)
+
+@Language.factory('asks tagger')
+class AsksTagger:
+    def __init__(self, name, nlp, language: str='en') -> None:
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+
+        self._language = language
+        self._matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
+        self._connectives = []
+        if language == 'en': # question words and questionmark
+            self._connectives = ['?', 'what', 'how', 'who', 'when']
+        else: # Support for future languages
+            pass
+
+        for con in self._connectives:
+            self._matcher.add(con, None, nlp(con))
+        
+
+    def __call__(self, doc: Doc) -> Doc:
+        matches = self._matcher(doc)
+        asks_spans = [doc[start:end] for _, start, end in matches]
+        doc._.asks_span_indices = [{'start': span.start, 'end': span.end, 'label': span.label} for span in filter_spans(asks_spans)] # Save the asks connectives found
+        
+        return doc

text_complexity_analyzer_cm/pipes/causal_connectives_tagger.py

@@ -0,0 +1,34 @@
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Doc
+from spacy.tokens import Span
+from spacy.util import filter_spans
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+causal_connectives_getter = lambda doc: [doc[span['start']:span['end']] for span in doc._.causal_connectives_span_indices]
+
+Doc.set_extension('causal_connectives_span_indices', force=False, default=[])
+Doc.set_extension('causal_connectives', force=False, getter=causal_connectives_getter)
+
+@Language.factory('causal connective tagger')
+class CausalConnectivesTagger:
+    def __init__(self, name, nlp, language) -> None:
+        self._language = language
+        self._matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
+        self.causal_connectives = []
+        if language == 'en':
+            self.causal_connectives = ['to repeat, briefly', 'finally', 'therefore', 'with this in mind', 'in conclusion', 'because of this', 'because of', 'as a consequence', 'to this end', 'on the score of', 'then', 'because', 'so', 'later', 'hence', 'in short', 'for this reason', 'thus', 'so much that', 'accordingly', 'for', 'so then', 'as I have said', 'therefore', 'in summary', 'on the whole', 'consequently', 'for this purpose', 'since', 'as a result', 'to sum up', 'so that', 'as you can see']
+        else: 
+            pass
+        for con in self.causal_connectives:
+            self._matcher.add(con, None, nlp(con))
+        
+    def __call__(self, doc: Doc) -> Doc:
+        matches = self._matcher(doc)
+        causal_connectives_spans = [doc[start:end] for _, start, end in matches]       
+        doc._.causal_connectives_span_indices = [{'start': span.start,
+                                                'end': span.end,
+                                                'label': span.label}
+                                                for span in filter_spans(causal_connectives_spans)]
+        return doc

text_complexity_analyzer_cm/pipes/emphatics_tagger.py

@@ -0,0 +1,42 @@
+from spacy.matcher import PhraseMatcher
+from spacy.tokens import Doc
+from spacy.tokens import Span
+from spacy.util import filter_spans
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+emphatics_getter = lambda doc: [doc[span['start']:span['end']]
+                                        for span in doc._.emphatics_span_indices]
+
+Doc.set_extension('emphatics_span_indices', force=False, default=[])
+Doc.set_extension('emphatics', force=False, getter=emphatics_getter)
+
+@Language.factory('emphatics tagger')
+class EmphaticsTagger:
+    def __init__(self, name, nlp, language) -> None:
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+
+        self._language = language
+        self._matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
+        self._connectives = []
+        if language == 'en': # emphatics connectives for spanish
+            self._connectives = ['him', 'there', 'their', 'it', 'he', 'she', 'we', 'who', 'them', 'they', 'you', 'himself', 'her', 'whom', 'itself', 'somebody', 'something', 'us', 'anybody', 'herself', 'anyone', 'everybody', 'nobody', 'everyone', 'themselves', 'yourself', 'someone', 'his', 'yours']
+        else: # Support for future languages
+            pass
+
+        for con in self._connectives:
+            self._matcher.add(con, None, nlp(con))
+        
+
+    def __call__(self, doc: Doc) -> Doc:
+        matches = self._matcher(doc)
+        emphatics_spans = [doc[start:end] for _, start, end in matches]
+
+        doc._.emphatics_span_indices = [{'start': span.start,
+                                                    'end': span.end,
+                                                    'label': span.label}
+                                                for span in filter_spans(emphatics_spans)] # Save the emphatics connectives found
+        
+        return doc

text_complexity_analyzer_cm/pipes/feature_counter.py

@@ -0,0 +1,32 @@
+from itertools import tee
+from spacy.tokens import Doc
+from spacy.tokens import Token
+from spacy.language import Language
+
+from text_complexity_analyzer_cm.constants import ACCEPTED_LANGUAGES
+
+Doc.set_extension('feature_count', default=None, force=True)
+
+@Language.factory('feature counter')
+class FeatureCounter:
+
+    def __init__(self, nlp, name, language) -> None:
+        if not language in ACCEPTED_LANGUAGES:
+            raise ValueError(f'Language {language} is not supported yet')
+        self.language = language
+        self.counter_function = None
+
+
+    def __call__(self, doc: Doc) -> Doc:
+        '''
+        This method will calculate the the 'counter_function' on a text. Said function will be handle different counting.
+
+        Parameters:
+        doc(Doc): A Spacy document.
+        '''
+        if self.counter_function is None:
+            raise AttributeError('No function to count features was provided.')
+        # Prepare iterators to extract previous and current sentence pairs.
+        doc._.feature_count = self.counter_function(doc)
+
+        return doc