add source files

# Specify latent true scores

@@ -0,0 +1,46 @@
+# Specify latent true scores 
+lx1 =~ 1 * x1 
+lx2 =~ 1 * x2 
+lx3 =~ 1 * x3 
+
+# Specify mean of latent true scores 
+lx1 ~ gamma_lx1 * 1 
+lx2 ~ 0 * 1 
+lx3 ~ 0 * 1 
+# Specify variance of latent true scores 
+lx1 ~~ sigma2_lx1 * lx1 
+lx2 ~~ 0 * lx2 
+lx3 ~~ 0 * lx3 
+# Specify intercept of obseved scores 
+x1 ~ 0 * 1 
+x2 ~ 0 * 1 
+x3 ~ 0 * 1 
+# Specify variance of observed scores 
+x1 ~~ sigma2_ux * x1 
+x2 ~~ sigma2_ux * x2 
+x3 ~~ sigma2_ux * x3 
+# Specify autoregressions of latent variables 
+lx2 ~ 1 * lx1 
+lx3 ~ 1 * lx2 
+# Specify latent change scores 
+dx2 =~ 1 * lx2  
+dx3 =~ 1 * lx3  
+# Specify latent change scores means  
+dx2 ~ 0 * 1  
+dx3 ~ 0 * 1  
+# Specify latent change scores variances  
+dx2 ~~ 0 * dx2  
+dx3 ~~ 0 * dx3  
+# Specify constant change factor  
+g2 =~ 1 * dx2 + 1 * dx3  
+# Specify constant change factor mean  
+g2 ~ alpha_g2 * 1  
+# Specify constant change factor variance  
+g2 ~~ sigma2_g2 * g2  
+# Specify constant change factor covariance with the initial true score  
+g2 ~~ sigma_g2lx1 * lx1 
+# Specify proportional change component  
+dx2 ~ beta_x * lx1  
+dx3 ~ beta_x * lx2  
+# Specify autoregression of change score  
+dx3 ~ phi_x * dx2  

.gitignore

@@ -0,0 +1,8 @@
+ECCE_analysis
+ECCE_texts
+ICNALE_analysis
+ICNALE_texts
+results
+inputtexts
+.DS_Store
+

README.md

@@ -1,14 +1,15 @@
 ---
-title: Engagement Analyzer Demo3
-emoji: 👁
-colorFrom: blue
-colorTo: purple
+title: Engagement analyzer (demo)
+emoji: 👀
+colorFrom: gray
+colorTo: red
 sdk: streamlit
-sdk_version: 1.39.0
-app_file: app.py
+sdk_version: 1.25.0
+app_file: demo.py
 pinned: false
-license: cc-by-nc-4.0
-short_description: demo 3
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# engagement-analyzer-demo
+
+This is a demo of automatic analysis tool for Engagement (Martin & White, 2005).
+

analyzer.py

@@ -0,0 +1,147 @@
+import re
+import os
+import spacy_streamlit
+from collections import Counter
+import glob
+
+import spacy
+from spacy.tokens import Doc
+from spacy.cli._util import import_code
+
+from utils.visualize import visualize_spans
+from utils.utility import preprocess, delete_overlapping_span, cleanup_justify
+
+from resources.text_list import TEXT_LIST
+from resources.template_list import TPL_SPAN, TPL_SPAN_SLICE, TPL_SPAN_START
+from resources.colors import COLORS_1
+
+
+from skbio import diversity as dv
+
+from pipeline.post_processors import simple_table, const_table, ngrammar, diversity_values
+import pandas as pd
+
+# from pipeline.custom_functions import custom_functions
+SPAN_ATTRS = ["text", "label_", "start", "end"]
+CATEGORIES = ['ATTRIBUTION', "CITATION", "COUNTER", "DENY", "ENDOPHORIC", "ENTERTAIN", "JUSTIFYING", "MONOGLOSS", "PROCLAIM", "SOURCES"]
+
+
+# spacy.prefer_gpu()
+
+def load_model(spacy_model):
+    # source = spacy.blank("en")
+    nlp = spacy.load(spacy_model)  # , vocab=nlp_to_copy.vocab
+    nlp.add_pipe('sentencizer')
+    return (nlp)
+
+# source = spacy.blank("en")
+
+modelname = "en_engagement_LSTM_f3"
+# modelname = "en_engagement_LSTM_f5"
+# modelname = "en_engagement_Dual_RoBERTa_acad3_f4"
+
+os.makedirs(os.path.join("/Users/masakieguchi/Dropbox/0_Projects/0_basenlp/SFLAnalyzer/engagement-analyzer-demo/results", modelname))
+
+import_code("pipeline/custom_functions.py")
+
+# nlp = spacy.load("en_engagement_three_RoBERTa_base_LSTM384")
+nlp = spacy.load(modelname)
+# doc = nlp(preprocess(TEXT_LIST[0]))
+
+# cleanup_justify(doc, doc.spans["sc"])
+# delete_overlapping_span(doc.spans['sc'])
+
+# data, cols = const_table(doc, spans_key='sc', attrs=SPAN_ATTRS)
+# seq = [s for s in doc.spans["sc"]]
+# span_ngrams = ngrammar(seq=seq, n=3)
+
+# df = pd.DataFrame(data, columns=cols)
+
+# constant_value = 42
+# new_col = pd.Series([constant_value] * df.shape[0], name='new_col')
+
+# doclen = len(doc)
+# doc_len = pd.Series([doclen] * df.shape[0], name='nwords')
+
+# df.insert(0, "new", new_col, True)
+# df.insert(1, "nwords", doc_len, True)
+
+# df.to_csv("results/test.csv")
+
+
+
+# inputfiles = glob.glob("ECCE_texts/preprocessed/*.txt")
+inputfiles = glob.glob("ICNALE_texts/*/*.txt")
+savedir = "ICNALE_analysis"
+storeall = True
+storage = []
+os.makedirs(os.path.join("ICNALE_analysis", modelname))
+
+
+doc_level_storage = []
+
+for file in inputfiles:
+
+    filename = os.path.split(file)[-1]
+
+    with open(file, "r") as f:
+        text = f.read()
+
+    text = preprocess(text)
+    doc = nlp(text)
+    cleanup_justify(doc, doc.spans["sc"])
+    delete_overlapping_span(doc.spans['sc'])
+
+    data, cols = const_table(doc, spans_key='sc', attrs=SPAN_ATTRS)
+    seq = [s for s in doc.spans["sc"]]
+    span_ngrams = ngrammar(seq=seq, n=3)
+
+
+    ### Make it a dataset
+    df = pd.DataFrame(data, columns=cols)
+    df = df.astype({"start": int, "end": int}) #convert col type
+    df = df.sort_values(by= ['start']) #and sort by start
+    # constant_value = 42
+    new_col = pd.Series([filename] * df.shape[0], name='filename')
+
+    doclen = len(doc)
+    doc_len = pd.Series([doclen] * df.shape[0], name='nwords')
+
+    df.insert(0, "filename", new_col, True)
+    df.insert(1, "nwords", doc_len, True)
+    df.to_csv(f"{savedir}/{modelname}/{filename}.csv")
+
+    sequences = list(df['label_'])
+    # Engagement ngrams
+    span_bigrams = ngrammar(seq=seq, n=2)
+    bidf = pd.DataFrame(span_bigrams)
+
+    # constant_value = 42
+    new_col = pd.Series([filename] * bidf.shape[0], name='filename')
+    bidf = bidf.insert(0, "filename", new_col, True)
+
+
+    ## Document level 
+    doc_level = {}
+    counts = df['label_'].value_counts().reindex(CATEGORIES, fill_value=0)
+    div = diversity_values(list(counts))
+
+    div_data = pd.DataFrame.from_dict(div, orient='index')
+    
+    doc_data = pd.concat([counts, div_data], axis = 0).T
+    doc_data.insert(0, "filename", filename, True)
+    doc_data.insert(1, "nwords", doc_len, True)
+    doc_data.to_csv(f"{savedir}/{modelname}/ddata_{filename}.csv")
+
+    if storeall:
+        storage.append(df)
+        doc_level_storage.append(doc_data)
+
+
+alldf = pd.concat(storage)
+
+alldf.to_csv(f"{savedir}/0_{modelname}_20230726.csv")
+
+
+# alldoc = pd.concat(doc_level_storage)
+# alldoc.to_csv(f"{savedir}/1_{modelname}_doc_20230426.csv")

demo.py

@@ -0,0 +1,307 @@
+import re
+# import spacy_streamlit
+# from collections import Counter
+
+import spacy
+# from spacy.tokens import Doc
+
+# from spacy_streamlit import visualize_spans
+
+import streamlit as st
+
+from utils.utility import delete_overlapping_span, cleanup_justify
+from utils.visualize import visualize_spans
+
+# nlp = spacy.load(
+#     "packages/en_engagement_RoBERTa-0.0.2/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.2"
+# )
+
+#Load from local storage
+#MODEL_LIST = ['en_engagement_RoBERTa-ME-AtoE.tar.gz']
+
+#model = st.selectbox('Select model', MODEL_LIST, index=0)
+#nlp = spacy.load("packages/" + model)
+
+# Load from huggingface
+# sm = spacy.load('en_core_web_sm', disable=['ner'])
+
+st.set_page_config(page_title="ENGAGEMENT analyzer (beta ver 0.3)",
+                   layout="wide",
+                   initial_sidebar_state="expanded")
+
+
+@st.cache(allow_output_mutation=True)
+def load_model():
+    # nlp = spacy.load("en_engagement_RoBERTa_context_flz")
+    nlp = spacy.load("en_engagement_LSTM")
+    # nlp = spacy.load("en_engagement_spl_RoBERTa_base_attention")
+    return (nlp)
+
+
+nlp = load_model()
+
+doc = nlp(
+    'Welcome! Probably this is one of the few attempts to teach a machine how to read the discourse...! Although it is not perfect, you should be able to get a good place to start for your stance-taking analyses. The result will be presented here.'
+)
+
+# TPL_ENT = """
+# <mark class="entity" style="background: {bg}; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;">
+#     {text}
+#     <span style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem">{label}</span>
+# </mark>
+# """
+
+TPL_SPANS = """
+<div class="spans" style="line-height: 4.5;">
+    {text}
+    {span_slices}
+    {span_starts}
+</div>
+"""
+
+TPL_SPAN = """
+<span style="font-weight: bold; display: inline-block; line-height: 3; padding-bottom: 12px;position: relative;">
+    {text}
+    {span_slices}
+    {span_starts}
+</span>
+"""
+
+TPL_SPAN_SLICE = """
+<span style="background: {bg}; top: {top_offset}px;  display: inline-block; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
+</span>
+"""
+
+TPL_SPAN_START = """
+<span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+    <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+       
+        {label}{kb_link}
+    </span>
+</span>
+
+"""
+
+# TPL_SPAN_START_RTL = """
+# <span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+#     <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+#         {label}{kb_link}
+#     </span>
+# </span>
+# """
+
+DEFAULT_TEXT = """Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia."""
+
+TEXT_LIST = [
+    '''To a significant extent, individuals can be considered responsible for the rise of Hitler to power on the 31st of January, 1933. Hitler himself, the charismatic leader of the Nazi Party, as well as creator of Nazi policy, played a key role in his own rise to power. However, other individuals in government, such as Hindenburg and von Papen were influential in Hitler’s rise. To a small extent, other factors also enabled Hitler to rise to power such as the Depression and the weakness of the political system. Nevertheless to a significant extent, individuals can be held responsible for the rise of Adolf Hitler to power.''',
+    '''Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia.''',
+    '''Certainly, the argumentation is not without some faults. For example, the statement that “linking homosexuality to witches fulfills the same purpose” is not supported by references to the readings. It is not clear who was linking homosexuality to witches and in what context. Nevertheless, overall and in line with the general tendencies reported in the previous section, the author employs various contracting and expanding engagement resources successfully. However, a large part of the successful use of engagement resources seems to be related to how the author structures these strategies throughout the text, namely in a wave-like fashion: from acknowledging the opinions of others, to countering them by offering one’s own interpretation, to supporting it by acknowledging other sources.''',
+    '''As the centuries passed, accounts of witchcraft became more and more specific; details of witches’ ceremonies and oaths became more concrete and whatever the condemned humans confessed to was treated as fact. As discussants correctly pointed out, Bernardino of Siena, Martin Le Franc, and the anonymous author of the Errores Gazariorum all have an even more aggressive campaign against witches than did the authors of our previous readings. By depicting their rituals and customs, they look to paint the most grotesque picture of witches possible. Their frenzied accusations, were some of the main catalysts of the subsequent witch hunts.''',
+    '''The post labeled “Witchcraft as a Problem in Society” clearly explains the contribution that each text makes to the witch hunts. While two of the authors focused on describing, in full detail, the shocking and disturbing practices that witches partook of, the others tried to prove that the witch threat was real. These last texts sought to explain witchcraft so as to convince readers that witches actually existed. As all posts reiterate, the devil is definitely at the source of witchcraft.''',
+    '''The third part temporarily puts aside mediation analysis and shifts the discussion to moderation analysis. In Chapter 7, I show how a multiple regression model can be made more flexible by allowing one variable’s effect to depend linearly on another variable in the model. The resulting moderated multiple regression model allows an investigator to ascertain the extent to which X’s influence on outcome variable Y is contingent on or interacts with a moderator variable W.''',
+    '''For instance, research has shown that people have a tendency to justify close others’ unethical actions to protect them (Gino and Galinsky 2012). Research has also shown that parents who feel close to their children often adopt strict curfew practices (Elder et al., 1995). (EC-33)''',
+    '''Fitzpatrick and Pagani (2013) found that engagement skills in classroom behaviour at kindergarten were related with better math scores and academic success. (LC-0525-EN)''',
+    '''The COAG Reform Council (2013) indicated that when compared to other students, Australian Year 4 students who attended one year of ECEC services or programs gained 11 points higher in reading (LC-0471-MA). Preliminary evidence suggests that teaching children from low-income families using humanoid robots increases motivation, sense of community, and self-expression... (EC-64). These findings suggest that visual perception takes up only a small fraction of fixation durations. Specifically, Verdelhan (2010) proposes a two-country, one-good model in which each country has an exogenously specified i.i.d. consumption growth process. Waters & Baur (2003) suggest that children or adolescents who are overweight or obese suffer from social and psychological issues. (LC-0460-EN)''',
+    '''According to the Australian Bureau of Statistics (2008), the percentage of obese or overweight adults is a staggering 60%.
+    According to George et al. (2011), in the UK immigration has improved the academic performance of the native children.
+    According to UNICEF (2011) a child that is breastfed within the first hour of life is fourteen times less likely to die from diarrhoea or pneumonia.''',
+    '''As far as I am concerned, I do think globalization is good chance for China’s developing. From my point of view, I prefer to think that advantages of globalization outweighs disadvantages. ''',
+    '''As we know, China has made great progress for these years. I think it is the result of globalization. We all know China is a fast-developing country. We can seethe great progress that China has made. ''',
+    '''His idea was that an important ninth century bishop called John Anglicus may indeed have given birth to a child in full view of everyone on the streets of Rome, but that this bishop was not and never had been the pope. Of course, there is no evidence whatever for this, as Leibnitz himself well knew.''',
+    '''On the whole, however, when evaluating meanings metaphorically, the Chinese EFL learners hedge and qualify their statements subjectively, tempering the certainty and authority of their assertions rather than using the resources of interpersonal metaphor to reinforce and substantiate their arguments. These tendencies reveal a key area for pedagogical intervention. Namely, instruction could focus on the value of construing metaphors objectively to obscure the author as the source of the evaluation. Similarly, raising students’ awareness of the space of negotiation and the value of offering assertions on a cline of certainty (e.g., IT IS EVIDENT) rather than through exclusive declarations of shared knowledge (e.g., AS WE ALL KNOW) is critical for academic writing refinement. Instructional interventions such as these are key areas for further investigation.''',
+    '''Of the defendants involved in Utah Pie Company’s case only one seems to have emerged as exceptionally successful. However this success was not a factor of overwhelming market power, as can be seen by the dominant position of Mrs. Smith’s during this time, which had maintained a 39-45 percent market share over the corresponding period.''',
+    '''Because of the evidence presented by Tremblay and Tremblay, it would appear that mergers in the brewing industry would have been procompetitive because of economies of scale. However, allowing a firm to acquire more than 20% of the market in Wisconsin would give it too much power to charge higher prices, even if the merger would help lower total average costs.''',
+    '''Taken in whole, the economic evidence for grocery retailers in the decades after the Von’s decision suggests that increased concentration is pro-competitive and good for consumers, running contrary to the fears proposed by the Court.''',
+    '''The remedies that Justice Lewis Powell prescribed did not gain the desired effect, and I feel that they were not very effective in promoting competition. (Elan, S86)''',
+    '''There is the possibility for abuse if the producer sets different maximum prices for different retailers, allowing some to reap higher profits.''',
+    '''Such a program, with appropriate limits, would provide a balanced structure that would ensure quality patient care.''',
+    '''A recent survey of physician satisfaction by Harvard Medical School found that physician autonomy and the ability to provide high-quality care, not income, are most strongly associated with changes in job satisfaction . Thus, it seems reasonable to assume that health care providers would take advantage of the greater bargaining power to improve the quality of care. (Ken, S78-79)''',
+    '''It appears, then, that maximum price fixing does the greatest harm when set below a competitive level [evidentialize]. In Case 4 it could potentially do harm to small retailers trying to enter the market [suggest], but does so for the benefit of consumers and the producer. Based purely on the models, it appears that, at the very least, maximum prices deserve a Rule of Reason approach to evaluate their cost and benefits.''',
+    '''It could be seen that for this 68% of the respondents, Tampines was characteristically a location that provided for them all their basic needs. It can be seen from chart [11] that many people quoted accessibility and proximity to home, and even shopping as one of the ideal factors that drew them there. Accessibility is quite a key factor because it is evident that the regional centre was built on the basis of good infrastructure. In comparison, 32% of the respondents felt that the conventional downtown was still a major attraction, even though the regional centre had gained quite a vast amount of popularity and did to large extent have an air of modernity.'''
+]
+
+
+@st.cache(suppress_st_warning=True)
+def preprocess(text):
+    text = re.sub("\n\n", ' &&&&&&&&#&#&#&#&', text)
+    text = re.sub('\n', ' ', text)
+    text = re.sub('\s+', " ", text)
+    text = re.sub('&&&&&&&&#&#&#&#&', '\n\n', text)
+    return text
+
+
+@st.cache(allow_output_mutation=True)
+def delete_span(span_sc: dict):
+    id_del = []
+    for n, spn in enumerate(span_sc, start=1):
+        # print(spn)
+        # print(spn.label_)
+        if len(list(spn.sents)) > 1:
+            id_del.append(n)
+        # print(len(list(spn.sents)))
+
+    for idx in id_del:
+        # print(idx)
+        del span_sc[idx]
+
+
+# st.markdown('''
+#     <style>
+#         .sidebar .sidebar-content {{
+#             width: 300px;
+#         }}
+#     </style>
+# ''',
+#             unsafe_allow_html=True)
+
+with st.sidebar:
+    st.markdown("""
+
+    ## Engagement moves analyzed in this tool (adapted from Martin & White, 2005).
+
+| Engagement moves    | Description                                                                                                                                                                                                                          |
+| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `Deny` (Contract -> disclaim)      | An utterance which invokes a contrary position but which at the same time rejects it directly. The contrary position is hence given very little dialogic space.                                                                      |
+| `Counter` (Contract -> disclaim)    | An utterance which expresses the present proposition as replacing and thus 'countering' another proposition which would have been expected.                                                                                          |
+| `Concur` (Contract -> proclaim)    | An utterance which shows writers' expectation/assumption that the putative readers will agree with the preposition and/or to have the same knowledge.                                                                                |
+| `Pronounce` (Contract -> proclaim) | An utterance which expresses a strong level of writer commitment through the author's explicit emphasis and interpolation, thereby closing down the dialogic space.                                                                  |
+| `Endorse` (Contract -> proclaim)   | An utterance which refers to external sources as warrantable, undeniable,  and/or reliable. It expresses the writer’s alignment with and endorsement of an attributed proposition. As such, the dialogic space is somewhat narrowed. |
+| `Entertain` (Expand)          | An utterance which indicates author's position but as only one possibility amongst others, thereby opening up dialogic space.                                                                                                        |
+| `Attribute` (Expand)          | An utterance which signifies dialogic space as the writer attributes the proposition to an external source.                                                                                                                          |
+| `Monogloss`           | An utterance which does not employ any value of engagement. Such an utterance ignores the dialogic potential in an utterance.                                                                                                        |
+
+    """)
+#For a more complete description of the category, visit [the annotation guideline](https://egumasa.github.io/engagement-annotation-project/3_Categories/)!!
+
+st.sidebar.markdown("""
+    Engagement Analyzer is developed by [Masaki Eguchi](https://masakieguchi.weebly.com).
+    
+    ### Acknowledgements:
+    
+    The development of this tool has been supported by the following grants:
+
+    - The TIRF Doctoral Dissertation Grant 2022 sponsored by the International Research Foundation for English Language Education (TIRF) 
+    - The NFMLTA-MLJ Doctoral Dissertation Writing Support Grant 2022 sponsored by the National Federation of Modern Language Teachers Associations (NFMLTA) 
+    - Duolingo English Test Doctoral Dissertation Award, 2022
+    - The Graduate Student Research Award sponsored by the Department of Linguistics, University of Oregon
+
+    I would also like to thank:
+    - Aaron Miller (Linguistics, University of Oregon) for corpus annotation
+    - Ryan Walker (Linguistics/Antholopology, University of Oregon) for corpus annotation
+    - Dr. Kristopher Kyle (Associate Professor in Linguistics, University of Oregon)
+    """)
+
+
+cc = '<a rel="license" href="http://creativecommons.org/licenses/by-nc/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by-nc/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc/4.0/">Creative Commons Attribution-NonCommercial 4.0 International License</a>.'
+
+st.sidebar.markdown(cc, unsafe_allow_html=True)
+
+st.header("Engagement Analyzer (beta ver 0.2)")
+st.write(
+    "Engagement Analyzer is a free tool that analyzes English texts for rhetorical strategies under the Engagement system framework (Martin & White, 2005). Martin and White (2005) propose two basic stance-taking strategies: expansion and contraction, which are in turn divided into finer-grained rhetorical strategies. The current tool allows you to analyze texts for a total of nine rhetorical strategies. The definitions of each category label can be found from the side bar"
+)
+
+with st.expander("See more explanation"):
+    st.markdown("""
+    According to Martin & White (2005), Engagement is about how the writer of a text takes stances on a topic of discussion by `Expanding (= open)` or `Contracting (= close)` the discourse for alternative viewpoints. 
+
+     **Expansion strategy** = Discourse moves which open-up the dialogic space; the speaker/writer actively makes allowances for dialogically alternative positions and voices. (e.g., `ENTERTAIN`, `ATTRIBUTE`)
+
+     **Contraction strategy** = Discourse moves which close down dialogic space; the speaker/writer acts to challenge, fend off or restrict other alternative positions and voices (e.g., `DENY`, `COUNTER`, `PRONOUNCE`, `ENDORSE`).
+
+    """)
+
+st.info('''Updated on Jan.11th, 2023\n
+The current version was trained on 2,519 sentences and tested on 443 sentences. It achieved the following benchmark:
+- Macro F1 = .75
+- Macro Precision = .78
+- Macro Recall = .74
+I expect that the model's performance improves as the annotated dataset gets larger.
+''')
+
+with st.form("my_form"):
+
+    st.subheader("Option 1: selecting example text from list")
+    text = st.selectbox('', TEXT_LIST)
+
+    st.subheader("Option 2: analyze your own text")
+    input_text = st.text_area(
+        label="",
+        value=
+        "I would strongly encourage you to put your texts here to analyze it for stance-taking expressions.",
+        height=120)
+    st.text(
+        'The text from the pull-down list and in the textbox cannot be analyzed at the same time. Please select the mode.'
+    )
+
+    textmode = st.radio(
+        label='Choose the mode.',
+        options=['Option 1: Pull-down choice', 'Option 2: My own text'],
+        index=1)
+
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        if textmode == 'Option 2: My own text':
+            text = input_text
+            with st.spinner('Analysis in progress...'):
+                doc = nlp(preprocess(text))
+            #st.markdown("> " + input_text)
+        else:
+            with st.spinner('Analysis in progress...'):
+                doc = nlp(preprocess(text))
+            #st.markdown("> " + text)
+
+## Dependency parsing
+
+# if textmode == 'My own text':
+#     text = input_text
+#     doc = nlp(preprocess(text))
+#     #st.markdown("> " + input_text)
+# else:
+#     doc = nlp(preprocess(text))
+#     #st.markdown("> " + text)
+
+# st.header("Text", "text")
+# st.write(text)
+# delete_span(doc.spans['sc'])
+
+cleanup_justify(doc, doc.spans['sc'])
+delete_overlapping_span(doc.spans['sc'])
+
+visualize_spans(
+    doc,
+    spans_key="sc",
+    displacy_options={
+        'template': {
+            "span": TPL_SPAN,
+            'slice': TPL_SPAN_SLICE,
+            'start': TPL_SPAN_START,
+        },
+        "colors": {
+            "ENTERTAIN": "#82b74b",
+            "DENY": '#c94c4c',
+            "COUNTER": "#eea29a",
+            "PRONOUNCE": "#92a8d1",
+            "ENDORSE": "#034f84",
+            "CITATION": "#b2b2b2",
+            # "MONOGLOSS": "#3e4444",
+            "ATTRIBUTE": "#f7786b",
+            "ATTRIBUTION": "#f7786b",
+            "PROCLAIM": "#92a8d1",
+            "CITATION": "#F8C471",
+            "SOURCES": "#F7DC6F",
+            "JUSTIFYING": "#2ECC71",
+            "ENDOPHORIC": "#FAD7A0"
+        },
+    },
+    simple = True
+)
+
+st.subheader("Bibliography")
+st.markdown("""
+* Chang, P., & Schleppegrell, M. (2011). Taking an effective authorial stance in academic writing: Making the linguistic resources explicit for L2 writers in the social sciences. _Journal of English for Academic Purposes, 10_ (3), 140–151. https://doi.org/10.1016/j.jeap.2011.05.005
+* Martin, J. R., & White, P. R. R. (2005). _The language of evaluation: Appraisal in English._ Palgrave Macmillan.
+* Ryshina-Pankova, M. (2014). Exploring academic argumentation in course-related blogs through ENGAGEMENT. In G. Thompson & L. Alba-Juez (Eds.), _Pragmatics & Beyond New Series (Vol. 242, pp. 281–302)_. John Benjamins Publishing Company. https://doi.org/10.1075/pbns.242.14rys
+* Wu, S. M. (2007). The use of engagement resources in high- and low-rated undergraduate geography essays. _Journal of English for Academic Purposes, 6_ (3), 254–271. https://doi.org/10.1016/j.jeap.2007.09.006
+
+""")

main.py

@@ -0,0 +1,217 @@
+import re
+import spacy_streamlit
+from spacy_streamlit import visualize_parser
+from collections import Counter
+
+import spacy
+import streamlit as st
+
+# try:
+#     from .scripts.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3
+# except ImportError:
+#     from pipeline.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3
+from spacy.tokens import Doc
+from spacy.cli._util import import_code
+
+from utils.visualize import visualize_spans
+from utils.utility import preprocess, delete_overlapping_span, cleanup_justify
+from resources.text_list import TEXT_LIST
+from resources.text_list_BAWE import TEXT_LIST_BAWE
+from resources.template_list import TPL_SPAN, TPL_SPAN_SLICE, TPL_SPAN_START
+from resources.colors import COLORS_1
+
+import_code("pipeline/custom_functions.py")
+st.set_page_config(page_title='Engagement model comparaer', layout="wide")
+
+# spacy.prefer_gpu()
+
+MODEL_LIST =['en_engagement_LSTM', 'en_engagement_LSTM']
+
+# MODEL_LIST = [
+#     'en_engagement_three_RoBERTa_base_LSTM384-0.9.2/en_engagement_three_RoBERTa_base_LSTM384/en_engagement_three_RoBERTa_base_LSTM384-0.9.2',
+#     'en_engagement_three_RoBERTa_acad3_db-0.9.2/en_engagement_three_RoBERTa_acad3_db/en_engagement_three_RoBERTa_acad3_db-0.9.2',
+#     'silver-sweep-34/model-best',
+#     'expert-sweep-4/model-best',
+#     'confused-sweep-6/model-best',
+#     'warm-sweep-20/model-best',
+#     "en_engagement_three_RoBERTa_base-1.10.0/en_engagement_three_RoBERTa_base/en_engagement_three_RoBERTa_base-1.10.0",
+#     "en_engagement_three_RoBERTa_acad_db-1.10.0/en_engagement_three_RoBERTa_acad_db/en_engagement_three_RoBERTa_acad_db-1.10.0",
+#     "en_engagement_para_RoBERTa_acad_db3-0.9.0/en_engagement_para_RoBERTa_acad_db3/en_engagement_para_RoBERTa_acad_db3-0.9.0",
+#     "en_engagement_para_RoBERTa_acad_LSTM2-0.9.0/en_engagement_para_RoBERTa_acad_LSTM2/en_engagement_para_RoBERTa_acad_LSTM2-0.9.0",
+#     "en_engagement_three_RoBERTa_acad_db3-0.9.1/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.1",
+#     "en_engagement_three_RoBERTa_acad_LSTM2-0.9.1/en_engagement_three_RoBERTa_acad_LSTM2/en_engagement_three_RoBERTa_acad_LSTM2-0.9.1",
+#     "en_engagement_three_RoBERTa_acad_db3-0.9.2/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.2",
+#     'en_engagement_spl_RoBERTa_acad_db-0.7.4/en_engagement_spl_RoBERTa_acad_db/en_engagement_spl_RoBERTa_acad_db-0.7.4',
+#     'en_engagement_spl_RoBERTa_acad_db3-0.9.0/en_engagement_spl_RoBERTa_acad_db3/en_engagement_spl_RoBERTa_acad_db3-0.9.0',
+#     'en_engagement_spl_RoBERTa_acad_LSTM-0.7.2/en_engagement_spl_RoBERTa_acad_LSTM/en_engagement_spl_RoBERTa_acad_LSTM-0.7.2',
+#     'en_engagement_spl_RoBERTa_acad_512',
+#     'en_engagement_spl_RoBERTa_acad',
+#     'en_engagement_spl_RoBERTa_exp-0.6.5/en_engagement_spl_RoBERTa_exp/en_engagement_spl_RoBERTa_exp-0.6.5',
+#     # 'en_engagement_spl_RoBERTa_acad-0.3.4.1221/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.3.4.1221',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.2.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1228',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.1.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.1.1228',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.2.1220/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1220',
+#     # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa-0.2.2.1210/en_engagement_spl_RoBERTa/en_engagement_spl_RoBERTa-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa_acad_max1_do02',
+#     # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.3.1210/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.3.1210',
+#     # 'en_engagement_spl_RoBERTa_acad_max1_do02',
+#     # 'en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5/en_engagement_spl_RoBERTa_sqbatch_RAdam/en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5',
+#     # 'en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4',
+#     # 'en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5/en_engagement_spl_RoBERTa_cx_max1_do2/en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5',
+#     # 'en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4',
+#     # 'en_engagement_RoBERTa_context_flz-20221125_0.1.4/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221125_0.1.4',
+#     # 'en_engagement_RoBERTa_context_flz-20221117_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221117_0.1.3',
+#     # 'en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3/en_engagement_spl_RoBERTa_acad_context_flz/en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3',
+#     # 'en_engagement_RoBERTa_context_flz-Batch2_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-Batch2_0.1.1',
+#     # 'en_engagement_RoBERTa_context_flz-20221113_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.3',
+#     # 'en_engagement_RoBERTa_context_flz-20221113_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.1',
+#     # 'en_engagement_RoBERTa-0.0.2/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.2',
+#     # 'en_engagement_RoBERTa_combined-Batch2Eng_0.2/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-Batch2Eng_0.2',
+#     # 'en_engagement_RoBERTa_acad-0.2.1/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.2.1',
+#     # # 'en_engagement_BERT-0.0.2/en_engagement_BERT/en_engagement_BERT-0.0.2',
+#     # # 'en_engagement_BERT_acad-0.0.2/en_engagement_BERT_acad/en_engagement_BERT_acad-0.0.2',
+#     # # 'en_engagement_RoBERTa_acad-0.0.2/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.0.2',
+#     # 'en_engagement_RoBERTa-0.0.1/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.1',
+#     # # ' en_engagement_RoBERTa_sent-0.0.1_null/en_engagement_RoBERTa_sent/en_engagement_RoBERTa_sent-0.0.1_null',
+#     # # 'en_engagement_RoBERTa_combined-0.0.1/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-0.0.1',
+#     # 'en_engagement_RoBERTa-ME_AtoE/en_engagement_RoBERTa/en_engagement_RoBERTa-ME_AtoE',
+#     # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.3',
+#     # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.2'
+# ]
+
+multicol = st.checkbox("Compare two models", value=False, key=None, help=None)
+
+model1 = st.selectbox('Select model option 1', MODEL_LIST, index=0)
+model2 = st.selectbox('Select model option 2', MODEL_LIST, index=1)
+
+if '/' in model1:
+    model1 = "packages/" + model1
+
+if '/' in model2:
+    model2 = "packages/" + model2
+
+
+@st.cache(allow_output_mutation=True)
+def load_model(spacy_model):
+    # source = spacy.blank("en")
+    nlp = spacy.load(spacy_model) #, vocab=nlp_to_copy.vocab
+    nlp.add_pipe('sentencizer')
+    return (nlp)
+
+# source = spacy.blank("en")
+nlp = load_model(model1)
+
+if multicol:
+    nlp2 = load_model(model2)
+
+
+text = st.selectbox('select sent to debug', TEXT_LIST_BAWE)
+
+input_text = st.text_area("", height=200)
+
+# Dependency parsing
+st.header("Text", "text")
+if len(input_text.split(" ")) > 1:
+    doc = nlp(preprocess(input_text))
+    if multicol:
+        doc2 = nlp2(preprocess(input_text))
+    # st.markdown("> " + input_text)
+else:
+    doc = nlp(preprocess(text))
+    if multicol:
+        doc2 = nlp2(preprocess(text))
+    # st.markdown("> " + text)
+
+clearjustify = st.checkbox(
+    "Clear problematic JUSTIFYING spans", value=True, key=None, help=None)
+
+delete_overlaps = st.checkbox(
+    "Delete overlaps", value=True, key=None, help=None)
+
+# combine = st.checkbox(
+#     "Combine", value=False, key=None, help=None)
+
+# import copy
+# def combine_spangroups(doc1, doc2):
+#     # new_doc = Doc.from_docs([doc1, doc2], ensure_whitespace=True)
+#     new_doc = copy.deepcopy(doc1)
+#     # type()
+#     new_doc.spans['sc'].extend(doc2.spans['sc'])
+
+#     return new_doc
+
+
+# if combine:
+#     new_doc = combine_spangroups(doc, doc2)
+#     visualize_spans(new_doc,
+#                     spans_key="sc",
+#                     title='Combined spans:',
+#                     displacy_options={
+#                         'template': {
+#                               "span": TPL_SPAN,
+#                             'slice': TPL_SPAN_SLICE,
+#                             'start': TPL_SPAN_START,
+#                         },
+#                         "colors": COLORS_1,
+#                     },
+#                     simple=False)
+
+if clearjustify:
+    cleanup_justify(doc, doc.spans['sc'])
+
+if delete_overlaps:
+    delete_overlapping_span(doc.spans['sc'])
+    if multicol:
+        delete_overlapping_span(doc2.spans['sc'])
+
+if not multicol:
+    visualize_spans(doc,
+                    spans_key="sc",
+                    title='Engagement Span Anotations 1',
+                    displacy_options={
+                        'template': {
+                              "span": TPL_SPAN,
+                            'slice': TPL_SPAN_SLICE,
+                            'start': TPL_SPAN_START,
+                        },
+                        "colors": COLORS_1,
+                    },
+                    simple=False)
+
+
+else:
+    col1, col2 = st.columns(2)
+
+    with col1:
+        visualize_spans(doc,
+                        spans_key="sc",
+                        title='Engagement Span Anotations 1',
+                        displacy_options={
+                            'template': {
+                                "span": TPL_SPAN,
+                                'slice': TPL_SPAN_SLICE,
+                                'start': TPL_SPAN_START,
+                            },
+                            "colors": COLORS_1,
+                        },
+                        simple=False)
+
+    with col2:
+        visualize_spans(doc2,
+                        spans_key="sc",
+                        title='Engagement Span Anotations 2',
+                        displacy_options={
+                            'template': {
+                                "span": TPL_SPAN,
+                                'slice': TPL_SPAN_SLICE,
+                                'start': TPL_SPAN_START,
+                            },
+                            "colors": COLORS_1,
+                        },
+                        simple=False)
+
+
+dep_options = {"fine_grained": True, "distance": 120}
+visualize_parser(doc, displacy_options=dep_options)

pipeline/custom_functions.py

@@ -0,0 +1,190 @@
+from functools import partial
+from pathlib import Path
+from typing import Iterable, Callable
+import spacy
+from spacy.training import Example
+from spacy.tokens import DocBin, Doc
+
+# make the factory work
+# from scripts.rel_pipe import make_relation_extractor
+
+# make the config work
+# from scripts.rel_model import create_relation_model, create_classification_layer, create_instances, create_tensors
+# from scripts.custom_comps.SpanCat_extention import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3, build_mean_max_reducer4
+
+from typing import List, Tuple, cast
+from thinc.api import Model, with_getitem, chain, list2ragged, Logistic
+from thinc.api import Maxout, Linear, concatenate, glorot_uniform_init, PyTorchLSTM
+from thinc.api import reduce_mean, reduce_max, reduce_first, reduce_last
+from thinc.types import Ragged, Floats2d
+
+from spacy.util import registry
+from spacy.tokens import Doc
+from spacy.ml.extract_spans import extract_spans
+
+# @registry.layers("spacy.LinearLogistic.v1")
+# def build_linear_logistic(nO=None, nI=None) -> Model[Floats2d, Floats2d]:
+#     """An output layer for multi-label classification. It uses a linear layer
+#     followed by a logistic activation.
+#     """
+#     return chain(Linear(nO=nO, nI=nI, init_W=glorot_uniform_init), Logistic())
+
+
+@registry.layers("mean_max_reducer.v1.5")
+def build_mean_max_reducer1(hidden_size: int,
+                            dropout: float = 0.0) -> Model[Ragged, Floats2d]:
+    """Reduce sequences by concatenating their mean and max pooled vectors,
+    and then combine the concatenated vectors with a hidden layer.
+    """
+    return chain(
+        concatenate(
+            cast(Model[Ragged, Floats2d], reduce_last()),
+            cast(Model[Ragged, Floats2d], reduce_first()),
+            reduce_mean(),
+            reduce_max(),
+        ),
+        Maxout(nO=hidden_size, normalize=True, dropout=dropout),
+    )
+
+
+@registry.layers("mean_max_reducer.v2")
+def build_mean_max_reducer2(hidden_size: int,
+                            dropout: float = 0.0) -> Model[Ragged, Floats2d]:
+    """Reduce sequences by concatenating their mean and max pooled vectors,
+    and then combine the concatenated vectors with a hidden layer.
+    """
+    return chain(
+        concatenate(
+            cast(Model[Ragged, Floats2d], reduce_last()),
+            cast(Model[Ragged, Floats2d], reduce_first()),
+            reduce_mean(),
+            reduce_max(),
+        ), Maxout(nO=hidden_size, normalize=True, dropout=dropout),
+        Maxout(nO=hidden_size, normalize=True, dropout=dropout))
+
+
+# @registry.layers("mean_max_reducer.v2")
+# def build_mean_max_reducer2(hidden_size: int,
+#                             depth: int) -> Model[Ragged, Floats2d]:
+#     """Reduce sequences by concatenating their mean and max pooled vectors,
+#     and then combine the concatenated vectors with a hidden layer.
+#     """
+#     return chain(
+#         concatenate(
+#             cast(Model[Ragged, Floats2d], reduce_last()),
+#             cast(Model[Ragged, Floats2d], reduce_first()),
+#             reduce_mean(),
+#             reduce_max(),
+#         ), Maxout(nO=hidden_size, normalize=True, dropout=0.0),
+#         PyTorchLSTM(nO=64, nI=hidden_size, bi=True, depth=depth, dropout=0.2))
+
+
+@registry.layers("mean_max_reducer.v3")
+def build_mean_max_reducer3(hidden_size: int,
+                            maxout_pieces: int = 3,
+                            dropout: float = 0.0) -> Model[Ragged, Floats2d]:
+    """Reduce sequences by concatenating their mean and max pooled vectors,
+    and then combine the concatenated vectors with a hidden layer.
+    """
+    hidden_size2 = int(hidden_size / 2)
+    hidden_size3 = int(hidden_size / 2)
+    return chain(
+        concatenate(
+            cast(Model[Ragged, Floats2d], reduce_last()),
+            cast(Model[Ragged, Floats2d], reduce_first()),
+            reduce_mean(),
+            reduce_max(),
+        ),
+        Maxout(nO=hidden_size,
+               nP=maxout_pieces,
+               normalize=True,
+               dropout=dropout),
+        Maxout(nO=hidden_size2,
+               nP=maxout_pieces,
+               normalize=True,
+               dropout=dropout),
+        Maxout(nO=hidden_size3,
+               nP=maxout_pieces,
+               normalize=True,
+               dropout=dropout))
+
+
+@registry.layers("mean_max_reducer.v3.3")
+def build_mean_max_reducer4(hidden_size: int,
+                            depth: int) -> Model[Ragged, Floats2d]:
+    """Reduce sequences by concatenating their mean and max pooled vectors,
+    and then combine the concatenated vectors with a hidden layer.
+    """
+    hidden_size2 = int(hidden_size / 2)
+    hidden_size3 = int(hidden_size / 2)
+    return chain(
+        concatenate(
+            cast(Model[Ragged, Floats2d], reduce_last()),
+            cast(Model[Ragged, Floats2d], reduce_first()),
+            reduce_mean(),
+            reduce_max(),
+        ), Maxout(nO=hidden_size, nP=3, normalize=True, dropout=0.0),
+        Maxout(nO=hidden_size2, nP=3, normalize=True, dropout=0.0),
+        Maxout(nO=hidden_size3, nP=3, normalize=True, dropout=0.0))
+
+
+@registry.architectures("CustomSpanCategorizer.v2")
+def build_spancat_model(
+    tok2vec: Model[List[Doc], List[Floats2d]],
+    reducer: Model[Ragged, Floats2d],
+    scorer: Model[Floats2d, Floats2d],
+) -> Model[Tuple[List[Doc], Ragged], Floats2d]:
+    """Build a span categorizer model, given a token-to-vector model, a
+    reducer model to map the sequence of vectors for each span down to a single
+    vector, and a scorer model to map the vectors to probabilities.
+    tok2vec (Model[List[Doc], List[Floats2d]]): The tok2vec model.
+    reducer (Model[Ragged, Floats2d]): The reducer model.
+    scorer (Model[Floats2d, Floats2d]): The scorer model.
+    """
+    model = chain(
+        cast(
+            Model[Tuple[List[Doc], Ragged], Tuple[Ragged, Ragged]],
+            with_getitem(
+                0,
+                chain(tok2vec,
+                      cast(Model[List[Floats2d], Ragged], list2ragged()))),
+        ),
+        extract_spans(),
+        reducer,
+        scorer,
+    )
+    model.set_ref("tok2vec", tok2vec)
+    model.set_ref("reducer", reducer)
+    model.set_ref("scorer", scorer)
+    return model
+
+
+# @registry.architectures("spacy.SpanCategorizer.v1")
+# def build_spancat_model(
+#     tok2vec: Model[List[Doc], List[Floats2d]],
+#     reducer: Model[Ragged, Floats2d],
+#     scorer: Model[Floats2d, Floats2d],
+# ) -> Model[Tuple[List[Doc], Ragged], Floats2d]:
+#     """Build a span categorizer model, given a token-to-vector model, a
+#     reducer model to map the sequence of vectors for each span down to a single
+#     vector, and a scorer model to map the vectors to probabilities.
+#     tok2vec (Model[List[Doc], List[Floats2d]]): The tok2vec model.
+#     reducer (Model[Ragged, Floats2d]): The reducer model.
+#     scorer (Model[Floats2d, Floats2d]): The scorer model.
+#     """
+#     model = chain(
+#         cast(
+#             Model[Tuple[List[Doc], Ragged], Tuple[Ragged, Ragged]],
+#             with_getitem(
+#                 0,
+#                 chain(tok2vec,
+#                       cast(Model[List[Floats2d], Ragged], list2ragged()))),
+#         ),
+#         extract_spans(),
+#         reducer,
+#         scorer,
+#     )
+#     model.set_ref("tok2vec", tok2vec)
+#     model.set_ref("reducer", reducer)
+#     model.set_ref("scorer", scorer)
+#     return model

pipeline/post_processors.py

@@ -0,0 +1,889 @@
+
+from typing import List, Sequence, Tuple, Optional, Dict, Union, Callable
+import pandas as pd
+import spacy
+from spacy.language import Language
+from skbio import diversity as dv
+
+SPAN_ATTRS = ["text", "label_", "start", "end"]
+CATEGORIES = ['ATTRIBUTION', "CITATION", "COUNTER", "DENY", "ENDOPHORIC", "ENTERTAIN", "JUSTIFYING", "MONOGLOSS", "PROCLAIM", "SOURCES"]
+
+
+def simple_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
+                 spans_key: str = "sc",
+                 attrs: List[str] = SPAN_ATTRS):
+    columns = attrs + ["Conf. score"]
+    data = [
+        [str(getattr(span, attr))
+         for attr in attrs] + [score]  # [f'{score:.5f}']
+        for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores'])
+    ]
+    return data, columns
+
+
+# def span_info_aggregator()
+
+def construction_classifier(doc, span):
+    category = None
+    spanroot = span.root
+
+    ## Grabbing lexico-grammatical information
+    span_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in span]
+    span_dep = [t.dep_ for t in span]
+    span_token = [t.norm_ for t in span]
+    span_tag = [t.tag_ for t in span]
+
+
+    c = [c for c in spanroot.children]
+    c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]
+
+    c_norm = [c.norm_ for c in spanroot.children]
+    c_dep = [c.dep_ for c in spanroot.children]
+    c_pos = [c.pos_ for c in spanroot.children]
+    c_tag = [c.tag_ for c in spanroot.children]
+
+    right_dep = [c.dep_ for c in spanroot.rights]
+
+    #conditionals
+    subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+    argmentless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in spanroot.children)
+    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
+
+    ## nesting classifiers
+    if spanroot.dep_ == "conj":
+        while spanroot.dep_ == 'conj':
+            spanroot = spanroot.head
+    # if spanroot.dep_ == "poss":
+    #     while spanroot.dep_ == 'poss':
+    #         spanroot = spanroot.head
+
+    ## Conjunctions 
+    # Preconjunctions
+    if spanroot.dep_ in ['preconj', 'cc']:
+        category = "Conjunction"
+
+    ## NOUN PHRASES
+    # adverbial phrases
+    if spanroot.dep_ in ['amod']:
+        category = "Adjectival modifier"
+        # adverbial phrases
+    if spanroot.dep_ in ['compound']:
+        category = "Compound noun"
+
+    ## Nominal category
+    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
+        if "acl" in c_dep:
+            category = "Noun + Complement (Object)"
+        else:
+            category = "Object"
+
+    if spanroot.dep_ in ["nsubj", "nsubjpass"]:
+        if "acl" in c_dep:
+            category = "Noun + Complement (Subject)"
+        else:
+            category = "Subject"
+
+    ## ADJUNCTS
+    # prep phrases
+    if spanroot.dep_ in ['prep', 'agent']:
+        category = 'Prepositional phrase'
+    # adverbial phrases
+    if spanroot.dep_ in ['advmod', "npadvmod", "nmod", "npmod", 'quantmod']:
+        category = "Adverbial phrase"
+
+    ## Predication patterns
+    if spanroot.dep_ in ['acomp', 'oprd']:
+        if "xcomp" in c_dep:
+            category = "Subject predicate to-cl"
+        else:
+            category = "Adjectival complement"
+
+    if spanroot.dep_ in ['attr']:
+        subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+
+        c_head = [c.dep_ for c in spanroot.head.children]
+        if "expl" in c_head and "no_det" in span_t_dep_:
+            category = "There is/are no NOUN"
+        elif "expl" in c_head and spanroot.pos_ in ["NOUN"]:
+            category = "There is/are + Noun complement"
+        elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
+            category = "There is/are + Noun complement"
+        
+        elif spanroot.pos_ in ["NOUN", "PRON"]:
+            if "acl" in c_dep:
+                category = "Noun + Complement (attr)"
+            else:
+                category = "Nominal complement"
+
+        elif not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+            category = "Main verb 4"
+
+        elif spanroot.tag_ in ['NNP']:
+            category = "Nominal complement"
+
+
+    ####################################
+    ### clausal ####
+    ####################################
+    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", 'acl', 'pcomp', 'relcl']:
+
+        _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+        _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+        root_before_ccomp = [c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"]
+
+        _check_for_to = ["_".join([c.norm_, c.dep_])  for c in spanroot.subtree if c.head.dep_ == "advcl" and (c.dep_=="mark" or c.dep_ == "aux")]
+        entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+
+        ## Start with broad category, which is then re-evaluated for specific constructions.
+        if spanroot.dep_ in ['advcl', 'mark', 'acl', 'pcomp']:
+            ## Adverbial clauses
+            ### Finite-adverbial clauses
+            ### Non-finite adverbial clauses
+            subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+
+            if "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+                category = "Finite adverbial clause"
+            elif "mark" in span_dep and "aux" in span_dep :
+                category = "Finite adverbial clause"
+
+            elif "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"] and "expl" in c_dep:
+                category = "Finite adverbial clause"
+
+            elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
+                if spanroot.pos_ in ['VERB', "AUX"]:
+                    category = "Finite adverbial clause"
+
+                elif spanroot.pos_ not in ['VERB', "AUX"] and subjless:
+                    category = "Non-finite adv clause 1"
+
+                elif entire_cl:
+                    category = "Finite adverbial clause"
+
+            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+                # he doing his job
+                if argmentless:
+                    #e.g., frankly speaking, strictly speaking
+                    category = "Adverbial Phrase"
+                else:
+                    category = "Non-finite adv clause 2"
+
+            elif spanroot.pos_ not in ['VERB', "AUX"] and "mark" in span_dep and subjless:
+
+                category = "Non-finite adv clause 3"
+            
+            elif "aux" in c_dep and "TO" in c_tag:
+                category = "Adverbial Phrase"
+
+
+            elif "mark" not in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+                category = "Dependent Verb phrase"
+            
+            elif not argmentless:
+                category =  "Adverbial clause"
+            
+            elif spanroot.dep_ == "advcl":
+                category =  "Adverbial phrase"
+
+
+        if spanroot.dep_ in ['relcl', 'ccomp', 'acl']:
+
+            head = spanroot.head
+            if ";" in [t.norm_ for t in head.children]:
+                category = "Main verb 3"
+            elif "nsubj" not in span_dep:
+                category = "Dependent verb 1"
+            elif "mark" in span_dep:
+                category = "Complement clause"
+            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+                category = "Non-finite complement clause"
+            elif spanroot.dep_ in ['relcl']:
+                category = "Relative clause"
+            elif spanroot.dep_ in ['ccomp']:
+                category = "Complement clause"
+            elif spanroot.dep_ in ['acl']:
+                category = "Noun Complement clause"
+            else:
+                # print(_check_for_to)
+                category = "this one"
+
+        ## Specific constructions
+        # Extraposed that-clause or to-infinitives
+        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in ["VERB", "AUX"]:
+            print(c_dep)
+            if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
+                #eg it seems odd (oprd) that X.
+                #eg it is certain (acomp) that X.
+                category = "Extraposed that-cl (adj-complement)" #e.g., it is certain that X.
+
+            elif "xcomp" in c_dep or ("advcl" in c_dep):
+                if "for_mark" in _check_for_to:
+                    category = "Extraposed to-cl (explicit subj)" #eg It is possible to .
+                elif _check_to:
+                    category = "Extraposed to-cl 1" #eg It is possible to .
+                elif _check_ing:
+                    category = "Extraposed -ing 1" #eg It is possible to .
+            elif ("prep" in right_dep or "npadvmod" in right_dep) and "ccomp" in right_dep and spanroot.lemma_ == "be":
+                category = "Cleft construction" 
+
+            elif "attr" in c_dep:
+                category = "Extraposed that-cl (copula)" #eg It is a wonder that X.
+
+            else:
+                category = "Extraposed that-cl (VERB)" 
+
+        # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
+        #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
+        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "acomp" in c_dep:
+            if "xcomp" in c_dep:
+                if _check_to:
+                    category = "Extraposed to-cl 2" #eg it is difficult to decide.
+                elif _check_ing:
+                    category = "Extraposed -ing 2" #eg it is difficult to decide.
+            
+            else:
+                category = "Extraposed that-cl (adj-complement) 2"
+
+        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
+
+            category = "Extraposed that-cl (adj-complement) 3" #eg it seems odd that X.
+
+        
+        # something without dummy subject "it"
+        elif (("nsubj" in c_dep and spanroot.lemma_ in ['be']) or "nsubjpass" in c_dep) and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm:
+            
+            # store xcomp, if the head of the xcomp is acomp
+            _check_xcomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"]
+            _check_ccomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"]
+            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+            
+
+            if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
+                if any(root_before_ccomp):
+                    category = "Post-predicate that-cl"
+                else:
+                    category = "Comment clause"
+
+            elif ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in _check_ccomp:
+                category = "Post-predicate that-cl 2"
+
+            elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
+                category = "Post-predicate to-cl"
+
+            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_to:
+                category = "Subject predicate to-cl"
+
+            elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
+                category = "Subject predicate to-cl (passive)"
+
+            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_ing:
+                category = "Subject predicate -ing"
+            elif "ccomp" in c_dep:
+                category = "Subject predicate that-cl"
+            elif "acomp" in c_dep:
+                category = "Adjectival predicate"
+
+            elif "mark" in c_dep and ("nsubj" in c_dep or "nsubjpass" in c_dep):
+                category = "Finite-adverbial clause"
+            else:
+                category = "Main verb 1"
+
+        ## without dummy subject it, and lexical verbs
+        elif ("nsubj" in c_dep or "nsubjpass" in c_dep)  in c_dep and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm and spanroot.lemma_ not in ['be']:
+            _check_wh = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["attr", "advmod", 'dobj', 'nsubj'] and c.tag_ in ["WP", "WRB", "WDT", "WP$"]) and c.head.dep_ == "ccomp"]
+            _check_if = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"]) and c.head.dep_ == "ccomp"]
+
+            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+
+            if "ccomp" in c_dep and (_check_wh or _check_if):
+                category = "Post-predicate wh-cl"
+
+            elif "ccomp" in c_dep:
+                if any(root_before_ccomp):
+                    category = "Post-predicate that-cl"
+                else:
+                    category = "Comment clause"
+
+            elif "xcomp" in c_dep:
+                if _check_to:
+                    category = "Post-predicate to-cl"
+                elif _check_ing:
+                    category = "Post-predicate -ing"
+
+        # Existential
+        elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
+            category = "There is/are NOUN"
+            
+        elif "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]:
+            category = "Cleft construction"
+
+
+    if spanroot.dep_ in ['parataxis']:
+        if "_".join(span_dep) in ["nsubj_parataxis", "aux_parataxis", "nsubj_aux_parataxis"]:
+            category = "Comment clause"
+        else:
+            category = "parataxis (for now)"
+    
+
+    ## External comp
+    if spanroot.dep_ in ['xcomp']:
+        if spanroot.head.pos_ == 'ADJ' and "to_aux" in c_t_dep_:
+            category = "Adjective complement to-cl"
+        if spanroot.head.pos_ == 'VERB' and "to_aux" in c_t_dep_:
+            category = "Verb complement to-cl"
+    
+    if spanroot.dep_ in ['pcomp']:
+        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"] and 'ccomp' in c_dep:
+            category = "Participle + that-cl"
+        elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
+            category = "Participle"
+
+    ## Simple classifier
+    # if spanroot.dep_ in ['pcomp']:
+    #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
+    #         category = "Gerund"
+
+    if spanroot.dep_ in ['neg']:
+        category = "Negative particle"
+    if spanroot.dep_ in ['aux', 'auxpass']:
+        category = "Auxiliary"
+
+    # Modal verbs
+    if spanroot.tag_ == "MD":
+        category = "Modal auxiliary"
+
+
+    if spanroot.dep_ in ['dep', "csubj", 'csubjpass']:
+        if spanroot.head.dep_ in ['ROOT', 'ccomp'] and spanroot.head.pos_ in ['AUX', 'VERB'] and spanroot.pos_ in ['AUX', 'VERB']:
+            if spanroot.morph == spanroot.head.morph:
+                category = "Main verb 4"
+            else:
+                category = "Dependent verb 2"
+        elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
+            category = "Gerund"
+        elif spanroot.head.dep_ in ['conj', 'acl','relcl']:
+            if spanroot.morph == spanroot.head.morph:
+                category = "Main verb 4"
+            else:
+                category = "Dependent verb 2"
+        elif "VerbForm=Fin" in str(spanroot.morph):
+            category = "Dependent verb 2"
+
+    # Appositive phrases
+    if spanroot.dep_ in ['appos']:
+        if "nummod" in c_dep:
+            category = "Apposition"
+        elif spanroot.pos_ in ["PROPN"]:
+            category = "Appositive Proper Nouns"
+        elif spanroot.pos_ in ["NOUN"]:
+            category = "Appositive Noun Phrase"
+        elif spanroot.pos_ in ["VERB", "AUX"]:
+            _check = any(c.dep_ in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            if _check:
+                category = "Appositive Finite-clause"
+            
+    if spanroot.dep_ in ['appos', "dep", "attr"]:
+        if not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+            category = "Main verb 5"
+
+    if spanroot.dep_ in ["dep", "mark"]:
+        if spanroot.tag_ in ["RB", "IN", "CC"]:
+            category = "Conjunction"
+
+
+    #sometimes the extra-clausal links are not accurate
+    if spanroot.dep_ in ['aux', "auxpass", 'oprd', 'appos', "xcomp"]:
+        if spanroot.head.dep_ == "ROOT":
+            category = "Main verb"
+        else:
+            category = "dependent verb 5"
+
+    if span.label_ == "CITATION":
+        if "NNP" in span_tag or "NNPS" in span_tag:
+            if span_dep[0] == 'punct' and span_dep[-1] == 'punct':
+                category = "Parenthetical Citation"
+            elif span_tag[0] in ["NNP", "NNPS"]:
+                category = "Narrative Citation"
+        else:
+            category = "Other Citation"
+
+    if category == None:
+        category = spanroot.dep_
+
+    return category
+
+
+def construction_classifier2(doc, span):
+    category = None
+    spanroot = span.root
+
+    ## Grabbing lexico-grammatical information
+    span_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in span]
+    span_dep = [t.dep_ for t in span]
+    span_token = [t.norm_ for t in span]
+    span_tag = [t.tag_ for t in span]
+
+
+    c = [c for c in spanroot.children]
+    c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]
+
+    c_norm = [c.norm_ for c in spanroot.children]
+    c_dep = [c.dep_ for c in spanroot.children]
+    c_pos = [c.pos_ for c in spanroot.children]
+    c_tag = [c.tag_ for c in spanroot.children]
+
+    right_dep = [c.dep_ for c in spanroot.rights]
+
+    #conditionals
+    subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+    argmentless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in spanroot.children)
+    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
+    argless_span = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass', "dobj", 'ccomp', 'xcomp', 'dative', "attr", "oprd", "acomp"] for c in span)
+
+
+    ## nesting classifiers
+    if spanroot.dep_ == "conj":
+        while spanroot.dep_ == 'conj':
+            spanroot = spanroot.head
+
+    if spanroot.dep_ == "poss":
+        head = spanroot.head
+        if head.dep_ in ["pobj", "dobj", "obj", "iobj" , "dative"]:
+            category = "Posessive Noun (Object)"
+        elif head.dep_ in ["nsubj", "nsubjpass"]:
+            category = "Posessive Noun (Subject)"
+        else:
+            category = "Posessive Noun (Other)"
+
+
+    ## Conjunctions 
+    # Preconjunctions
+    if spanroot.dep_ in ['preconj', 'cc']:
+        category = "Conjunction"
+
+    ## NOUN PHRASES
+    # adverbial phrases
+    if spanroot.dep_ in ['amod']:
+        category = "Adjectival modifier"
+        # adverbial phrases
+    if spanroot.dep_ in ['compound']:
+        category = "Compound noun"
+
+    ## Nominal category
+    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj" , "dative"]:
+        if "acl" in c_dep:
+            category = "Noun + Complement (Object)"
+        else:
+            category = "Object"
+
+    if spanroot.dep_ in ["nsubj", "nsubjpass"]:
+        if "acl" in c_dep:
+            category = "Noun + Complement (Subject)"
+        else:
+            category = "Subject"
+
+    ## ADJUNCTS
+    # prep phrases
+    if spanroot.dep_ in ['prep', 'agent']:
+        category = 'Prepositional phrase'
+
+    # adverbial phrases
+    if spanroot.dep_ in ['advmod', "npadvmod", "nmod", "npmod", 'quantmod',  'nummod']:
+        category = "Adverbial phrase"
+
+    ## Predication patterns
+    if spanroot.dep_ in ['acomp', 'oprd']:
+        if "xcomp" in c_dep:
+            category = "Subject predicate to-cl"
+        else:
+            category = "Adjectival complement"
+
+    if spanroot.dep_ in ['attr']:
+        subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+
+        c_head = [c.dep_ for c in spanroot.head.children]
+        if "expl" in c_head and "no_det" in span_t_dep_:
+            category = "There is/are no NOUN"
+        elif "expl" in c_head and spanroot.pos_ in ["NOUN"]:
+            category = "There is/are + Noun complement"
+        elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
+            category = "There is/are + Noun complement"
+        
+        elif spanroot.pos_ in ["NOUN", "PRON"]:
+            if "acl" in c_dep:
+                category = "Noun + Complement (attr)"
+            else:
+                category = "Nominal complement"
+
+        elif not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+            category = "Main verb 4"
+
+        elif spanroot.tag_ in ['NNP']:
+            category = "Nominal complement"
+
+    ## External comp
+    if spanroot.dep_ in ['xcomp']:
+        if spanroot.head.pos_ == 'ADJ' and "to_aux" in c_t_dep_:
+            category = "Adjective complement to-cl"
+        if spanroot.head.pos_ == 'VERB' and "to_aux" in c_t_dep_:
+            category = "Verb complement to-cl"
+    
+    if spanroot.dep_ in ['pcomp']:
+        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"] and 'ccomp' in c_dep:
+            category = "Participle + that-cl"
+        elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
+            category = "Participle"
+
+    ## Simple classifier
+    # if spanroot.dep_ in ['pcomp']:
+    #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
+    #         category = "Gerund"
+
+    if spanroot.dep_ in ['neg']:
+        category = "Negative particle"
+    if spanroot.dep_ in ['aux', 'auxpass']:
+        category = "Auxiliary"
+
+    # Modal verbs
+    if spanroot.tag_ == "MD":
+        category = "Modal auxiliary"
+
+
+    ####################################
+    ### clausal ####
+    ####################################
+    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", 'acl', 'pcomp', 'relcl', 'punct']:
+
+        _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+        _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+        root_before_ccomp = [c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"]
+
+        _check_for_to = ["_".join([c.norm_, c.dep_])  for c in spanroot.subtree if c.head.dep_ == "advcl" and (c.dep_=="mark" or c.dep_ == "aux")]
+        entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+
+
+        ## Start with broad category, which is then re-evaluated for specific constructions.
+        if spanroot.dep_ in ['advcl', 'acl', 'punct', 'pcomp']: #'mark', 
+            ## Adverbial clauses
+            subjless = all(c.dep_ not in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            entire_cl = spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
+
+            ### Finite-adverbial clauses
+            if "mark" in span_dep and (spanroot.pos_ in ['VERB', "AUX"] or "aux" in span_dep ):
+                category = "Finite adverbial clause"
+
+            elif "mark" in span_dep and "aux" in span_dep :
+                category = "Finite adverbial clause"
+
+            elif "mark" in span_dep and spanroot.pos_ in ['VERB', "AUX"] and "expl" in c_dep:
+                category = "Finite adverbial clause"
+
+            elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
+                if spanroot.pos_ in ['VERB', "AUX"]:
+                    category = "Finite adverbial clause"
+
+                elif spanroot.pos_ not in ['VERB', "AUX"] and subjless:
+                    category = "Non-finite adv clause 1"
+
+                elif not argmentless:
+                    category =  "Finite adverbial clause"
+
+            ## non-finite
+            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+                # he doing his job
+                if argmentless:
+                    #e.g., frankly speaking, strictly speaking
+                    category = "Adverbial Phrase"
+                else:
+                    category = "Non-finite adv clause 2"
+
+            elif spanroot.pos_ not in ['VERB', "AUX"] and "mark" in span_dep and subjless:
+
+                category = "Non-finite adv clause 3"
+            
+            elif "aux" in c_dep and "TO" in c_tag:
+                category = "Adverbial Phrase"
+
+
+            elif "mark" not in span_dep and spanroot.pos_ in ['VERB', "AUX"]:
+                category = "Dependent Verb phrase"
+            
+            elif not argmentless:
+                category =  "Adverbial clause"
+            
+            elif spanroot.dep_ == "advcl":
+                category =  "Adverbial phrase"
+            
+            else:
+                category = "Finite adverbial clause "
+
+        if spanroot.dep_ in ['relcl', 'ccomp', 'acl', 'punct', "pcomp"]:
+            
+            head = spanroot.head
+            if ";" in [t.norm_ for t in head.children]:
+                category = "Main verb 3"
+
+            elif "nsubj" not in span_dep:
+                category = "Dependent verb 1"
+
+            elif "mark" in span_dep:
+                category = "Complement clause"
+            elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part", "Aspect=Perf|Tense=Past|VerbForm=Part"] and "aux" not in c_dep:
+                category = "Non-finite complement clause"
+            elif spanroot.dep_ in ['relcl']:
+                category = "Relative clause"
+            elif spanroot.dep_ in ['ccomp']:
+                category = "Complement clause"
+            elif spanroot.dep_ in ['acl']:
+                category = "Noun Complement clause"
+
+        ## Specific constructions
+        # Extraposed that-clause or to-infinitives
+        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in ["VERB", "AUX"]:
+            # print(c_dep)
+            if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
+                #eg it seems odd (oprd) that X.
+                #eg it is certain (acomp) that X.
+                category = "Extraposed that-cl (adj-complement)" #e.g., it is certain that X.
+
+            elif "xcomp" in c_dep or ("advcl" in c_dep):
+                if "for_mark" in _check_for_to:
+                    category = "Extraposed to-cl (explicit subj)" #eg It is possible to .
+                elif _check_to:
+                    category = "Extraposed to-cl 1" #eg It is possible to .
+                elif _check_ing:
+                    category = "Extraposed -ing 1" #eg It is possible to .
+            elif ("prep" in right_dep or "npadvmod" in right_dep) and "ccomp" in right_dep and spanroot.lemma_ == "be":
+                category = "Cleft construction" 
+
+            elif "attr" in c_dep:
+                category = "Extraposed that-cl (copula)" #eg It is a wonder that X.
+
+            else:
+                category = "Extraposed that-cl (VERB)" 
+
+        # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
+        #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
+        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "acomp" in c_dep:
+            if "xcomp" in c_dep:
+                if _check_to:
+                    category = "Extraposed to-cl 2" #eg it is difficult to decide.
+                elif _check_ing:
+                    category = "Extraposed -ing 2" #eg it is difficult to decide.
+            
+            else:
+                category = "Extraposed that-cl (adj-complement) 2"
+
+        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
+
+            category = "Extraposed that-cl (adj-complement) 3" #eg it seems odd that X.
+
+        
+        # something without dummy subject "it"
+        elif (("nsubj" in c_dep and spanroot.lemma_ in ['be']) or "nsubjpass" in c_dep) and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm:
+            
+            # store xcomp, if the head of the xcomp is acomp
+            _check_xcomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"]
+            _check_ccomp = [c.dep_ for c in spanroot.subtree if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"]
+            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+            
+
+            if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
+                if any(root_before_ccomp):
+                    category = "Post-predicate that-cl"
+                else:
+                    category = "Comment clause"
+
+            elif ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in _check_ccomp:
+                category = "Post-predicate that-cl 2"
+
+            elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
+                category = "Post-predicate to-cl"
+
+            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_to:
+                category = "Subject predicate to-cl"
+
+            elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
+                category = "Subject predicate to-cl (passive)"
+
+            elif "xcomp" in c_dep and spanroot.lemma_ in ['be'] and _check_ing:
+                category = "Subject predicate -ing"
+            elif "ccomp" in c_dep:
+                category = "Subject predicate that-cl"
+            elif "acomp" in c_dep:
+                category = "Adjectival predicate"
+
+            elif "mark" in c_dep and ("nsubj" in c_dep or "nsubjpass" in c_dep):
+                category = "Finite-adverbial clause"
+            elif not argmentless and "SCONJ" in c_pos:
+                category = "Finite-adverbial clause"
+            else:
+                category = "Main verb 1"
+
+        ## without dummy subject it, and lexical verbs
+        elif ("nsubj" in c_dep or "nsubjpass" in c_dep)  in c_dep and spanroot.pos_ in ["AUX", 'VERB'] and "it" not in c_norm and spanroot.lemma_ not in ['be']:
+            _check_wh = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["attr", "advmod", 'dobj', 'nsubj'] and c.tag_ in ["WP", "WRB", "WDT", "WP$"]) and c.head.dep_ == "ccomp"]
+            _check_if = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"]) and c.head.dep_ == "ccomp"]
+
+            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
+            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]
+
+            if "ccomp" in c_dep and (_check_wh or _check_if):
+                category = "Post-predicate wh-cl"
+
+            elif "ccomp" in c_dep:
+                if any(root_before_ccomp):
+                    category = "Post-predicate that-cl"
+                else:
+                    category = "Comment clause"
+
+            elif "xcomp" in c_dep:
+                if _check_to:
+                    category = "Post-predicate to-cl"
+                elif _check_ing:
+                    category = "Post-predicate -ing"
+
+
+
+        # Existential
+        elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
+            category = "There is/are NOUN"
+            
+        elif "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]:
+            category = "Cleft construction"
+
+        ### The end of clausal analysis
+    
+    if spanroot.dep_ in ['parataxis']:
+        if "_".join(span_dep) in ["nsubj_parataxis", "aux_parataxis", "nsubj_aux_parataxis"]:
+            category = "Comment clause"
+        else:
+            category = "Parataxis"
+    
+
+    if spanroot.dep_ in ['dep', "csubj", 'csubjpass']:
+        if spanroot.head.dep_ in ['ROOT', 'ccomp'] and spanroot.head.pos_ in ['AUX', 'VERB'] and spanroot.pos_ in ['AUX', 'VERB']:
+            if spanroot.morph == spanroot.head.morph:
+                category = "Main verb 4"
+            else:
+                category = "Dependent verb 2"
+        elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
+            category = "Gerund"
+        elif "VerbForm=Fin" in str(spanroot.morph) or "VerbForm=Inf" in str(spanroot.morph):
+            category = "Dependent verb 2"
+        elif spanroot.dep_ in ["csubj", 'csubjpass']:
+            category = "Dependent verb (csubj)"
+
+
+    # Appositive phrases
+    if spanroot.dep_ in ['appos']:
+        if "nummod" in c_dep:
+            category = "Apposition"
+        if spanroot.pos_ in ["PROPN"]:
+            category = "Appositive Proper Nouns"
+        elif spanroot.pos_ in ["NOUN"]:
+            category = "Appositive Noun Phrase"
+        elif spanroot.pos_ in ["VERB", "AUX"]:
+            _check = any(c.dep_ in ['nsubj', 'nsubjpass', 'csubj', 'csubjpass'] for c in spanroot.children)
+            if _check:
+                category = "Appositive Finite-clause"
+    
+
+    if spanroot.dep_ in ['appos', "dep", "attr"]:
+        if not subjless and spanroot.pos_ in ['VERB', "AUX"]:
+            category = "Main verb (likely parsing error)"
+
+    #sometimes the dep are on the conjunctions
+    if spanroot.dep_ in ["dep", "mark"]:
+        if spanroot.tag_ in ["RB", "IN", "CC"]:
+            category = "Conjunction"
+
+    if spanroot.dep_ in ["intj"]:
+        category = "Introjection"
+
+
+    #sometimes the extra-clausal links are not accurate
+    if spanroot.dep_ in ['aux', "auxpass", 'oprd', 'appos', "xcomp", "attr", 'dep', "meta", 'prt'] and category == None:
+        if spanroot.head.dep_ == "ROOT":
+            category = "Main verb"
+        else:
+            category = "dependent verb 5"
+
+    if span.label_ == "CITATION":
+        if "NNP" in span_tag or "NNPS" in span_tag:
+            if span_dep[0] == 'punct' and span_dep[-1] == 'punct':
+                category = "Parenthetical Citation"
+            elif span_tag[0] in ["NNP", "NNPS"]:
+                category = "Narrative Citation"
+        else:
+            category = "Other Citation"
+
+    if category == None:
+        category = spanroot.dep_
+
+    return category
+
+
+
+def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
+                spans_key: str = "sc",
+                attrs: List[str] = SPAN_ATTRS):
+    columns = attrs + ["Conf. score", "sent no.", "grammatical realization", 'span dep', "ner", 
+                       "POS", 'span dep seq', "TAG sequence", "POS sequence", "head", "head dep", "children", "morphology", "sent"]
+    data = []
+    # data = span_info_aggregator(doc, columns)
+    sentences = {s: i for i, s in enumerate(doc.sents)}
+
+    for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores']):
+
+        span_info = []
+        span_info.extend([str(getattr(span, attr)) for attr in attrs])
+
+        span_info.append(score)
+        span_info.append(int(sentences[span.sent]))
+        span_info.append(construction_classifier2(doc, span))
+        span_info.append(span.root.dep_)
+        span_info.append(span.root.ent_type_)
+        span_info.append(span.root.tag_)
+        span_info.append("_".join([t.dep_ for t in span]))
+        span_info.append("_".join([t.tag_ for t in span]))
+        span_info.append("_".join([t.pos_ for t in span]))
+        span_info.append(span.root.head.norm_)
+        span_info.append(span.root.head.dep_)
+        span_info.append("_".join([c.dep_ for c in span.root.children]))
+        span_info.append(span.root.morph)
+        span_info.append(span.sent.text.strip())
+
+        data.append(span_info)
+
+    return data, columns
+
+
+def ngrammar(seq: list, n=2, concat = False, sep = "-"):
+    result = []
+    n_item = len(seq)
+    for idx, item in enumerate(seq):
+        if idx + n <= n_item:
+            if concat:
+                result.append(sep.join(seq[idx: idx + n]))
+            else:
+                result.append(seq[idx: idx + n])
+    return result
+
+
+def diversity_values(count_vec: list):
+    result = {}
+    if len(count_vec) == 0:
+        count_vec = [0,0,0,0,0,0,0,0,0,0]
+
+    result['shannon'] = dv.alpha.shannon(list(count_vec), base=2)
+    result['brillouin_d'] = dv.alpha.brillouin_d(list(count_vec))
+    result["simpson_d"] = 1- dv.alpha.simpson(list(count_vec))    
+    result['simpson_e'] = dv.alpha.simpson_e(list(count_vec))
+    # result['gini_index'] = dv.alpha.gini_index(list(count_vec))
+    # result['faith_pd'] = dv.alpha.faith_pd(list(count_vec))
+
+    return result

requirements.txt

@@ -0,0 +1,25 @@
+pip
+spacy-streamlit #==1.0.4
+spacy>3.4.4
+# spacy-experimental==0.6.1
+# spacy-huggingface-hub==0.0.8
+# spacy-transformers==1.1.8
+# srsly==2.4.5
+scikit-bio
+# pip==23.3.1
+# setuptools
+# pydantic==1.* #necessary for spacy 3.4.4?
+# altair<5
+# streamlit
+typing_extensions<4.6.0
+
+
+# https://huggingface.co/egumasa/en_engagement_RoBERTa_combined/resolve/main/en_engagement_RoBERTa_combined-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_RoBERTa_context_flz/resolve/main/en_engagement_RoBERTa_context_flz-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_spl_RoBERTa_acad_max1_do02/resolve/main/en_engagement_spl_RoBERTa_acad_max1_do02-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_spl_RoBERTa_acad/resolve/main/en_engagement_spl_RoBERTa_acad-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_spl_RoBERTa_acad2/resolve/main/en_engagement_spl_RoBERTa_acad2-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_spl_RoBERTa_acad2/resolve/main/en_engagement_spl_RoBERTa_acad2-any-py3-none-any.whl
+# https://huggingface.co/egumasa/en_engagement_LSTM/resolve/main/en_engagement_LSTM-any-py3-none-any.whl
+https://huggingface.co/egumasa/en_engagement_LSTM/resolve/main/en_engagement_LSTM-any-py3-none-any.whl #This is the best in 2023
+# https://huggingface.co/egumasa/en_engagement_spl_RoBERTa_base_attention/resolve/main/en_engagement_spl_RoBERTa_base_attention-any-py3-none-any.whl

resources/colors.py

@@ -0,0 +1,16 @@
+COLORS_1 = {
+            "ENTERTAIN": "#82b74b",
+            "DENY": '#c94c4c',
+            "COUNTER": "#eea29a",
+            "PRONOUNCE": "#92a8d1",
+            "ENDORSE": "#034f84",
+            "CITATION": "#b2b2b2",
+            # "MONOGLOSS": "#3e4444",
+            "ATTRIBUTE": "#f7786b",
+            "ATTRIBUTION": "#f7786b",
+            "PROCLAIM": "#92a8d1",
+			"ENDOPHORIC": "#FAD7A0",
+			"SOURCES": "#F9E79F"
+			
+        }
+

resources/template_list.py

@@ -0,0 +1,48 @@
+TPL_ENT = """
+<mark class="entity" style="background: {bg}; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;">
+    {text}
+    <span style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem">{label}</span>
+</mark>
+"""
+
+TPL_SPANS = """
+<div class="spans" style="line-height: 4.5;">
+    {text}
+    {span_slices}
+    {span_starts}
+</div>
+"""
+
+TPL_SPAN = """
+<span style="font-weight: bold; display: inline-block; line-height: 3; padding-bottom: 12px;position: relative;">
+    {text}
+    {span_slices}
+    {span_starts}
+</span>
+"""
+
+TPL_SPAN_SLICE = """
+<span style="background: {bg}; top: {top_offset}px;  display: inline-block; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
+</span>
+"""
+
+TPL_SPAN_START = """
+<span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+    <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+       
+        {label}{kb_link}
+    </span>
+</span>
+
+"""
+
+TPL_SPAN_START_RTL = """
+<span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+    <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+        {label}{kb_link}
+    </span>
+</span>
+"""
+
+DEFAULT_TEXT = """Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia."""
+

utils/utility.py

@@ -0,0 +1,141 @@
+import re
+from collections import Counter
+from spacy.tokens import SpanGroup
+
+
+def preprocess(text):
+    text = re.sub("--- Para SEP ---", '\n', text)
+    text = re.sub("\n\n", ' &&&&&&&&#&#&#&#&', text)
+    text = re.sub('\n', ' ', text)
+    text = re.sub(r'\s+', " ", text)
+    text = re.sub('&&&&&&&&#&#&#&#&', '\n\n', text)
+    return text
+
+
+def del_spans(span_sc, indexes: list):
+
+    indexes.sort(
+        reverse=True
+    )  # reversing allows the deletion from the last, keeping the original index
+
+    for idx in indexes:
+        if idx + 1 < len(span_sc):
+            del span_sc[idx + 1]
+
+
+def delete_overlapping_span(span_sc: dict):
+    # print(span_sc)
+    start_token_list = [spn.start for spn in span_sc]
+    dict_ = Counter(start_token_list)
+    overlap = {k: v for k, v in dict_.items() if v > 1}
+
+    id_del = []
+    id_comp = {}
+
+    info = {}
+    for n, (spn, score) in enumerate(zip(span_sc, span_sc.attrs['scores']),
+                                     start=0):
+        res = {
+            'score': score,
+            'spn': spn,
+            'label': spn.label_,
+            'start': spn.start,
+            'end': spn.end,
+            'compare': spn.start in overlap,
+            "sents": len(list(spn.sents))
+        }
+        # print(res)
+        info[n] = res
+
+        if res['compare']:
+            if spn.start not in id_comp:
+                id_comp[spn.start] = n
+            else:
+                same_lbl = res['label'] == info[id_comp[spn.start]]['label']
+                update = res['score'] > info[id_comp[spn.start]]['score']
+                if update and same_lbl:
+                    print(res['label'], info[id_comp[spn.start]]['label'])
+                    print(same_lbl)
+                    id_del.append(id_comp[spn.start])
+                    id_comp[spn.start] = n
+                else:
+                    id_del.append(n)
+                # print(update)
+
+        # delete span beyond sentences
+        if len(list(spn.sents)) > 1:
+            id_del.append(n)
+
+    # print(id_comp)
+    del_spans(span_sc, id_del)
+    # for n, idx in enumerate(id_del):
+    #     # print(idx)
+
+    #     try:
+    #         del span_sc[idx - n]
+    #     except IndexError:
+    #         continue
+
+
+def cleanup_justify(doc, span_sc: dict):
+    # This function adjusts the JUSTIFYING span
+
+    # First create an index of span with JUSTIFYING tags
+    justifies = {}
+    for idx, span in enumerate(span_sc):
+        # temp_root = span.root
+        # while span.start <= temp_root.head.i <= span.end:
+        #     temp_root = temp_root.head
+        if span.label_ in ['JUSTIFYING']:
+            justifies[span.root] = {
+                "span": span,
+                "head": span.root.head,
+                "start": span.start,
+                "end": span.end,
+                "del": False,
+                "dependent": False,
+                "span_idx": idx
+            }
+    # print(justifies)
+
+    # flagging the dependency
+    for spanroot, info in justifies.items():
+        if spanroot.head in justifies:
+            info['dependent'] = True
+            info['del'] = True
+
+    # print(justifies)
+    new_spans = []
+    for spanroot, info in justifies.items():
+
+        if not info['dependent']:
+            # print("New Justifying candidate span:")
+            # print(doc[spanroot.left_edge.i:spanroot.right_edge.i + 1])
+
+            new_span = doc[spanroot.left_edge.i:spanroot.right_edge.i + 1]
+            new_span.label_ = "JUSTIFYING"
+
+            if new_span not in span_sc:
+                new_spans.append(new_span)
+                info['del'] = True
+
+        else:
+            info['del'] = True
+
+    to_delete = [
+        info['span_idx'] for spanroot, info in justifies.items() if info['del']
+    ]
+
+    to_delete_span = [
+        info['span'] for spanroot, info in justifies.items() if info['del']
+    ]
+
+    # print(to_delete)
+    # print(to_delete_span)
+
+    del_spans(span_sc, to_delete)
+
+    span_grp = SpanGroup(doc, spans=new_spans)
+    span_sc.extend(span_grp)
+
+    # print(justifies)

utils/visualize.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#
+# This code is adapted from spacy-streamlit package by explosion
+# https://github.com/explosion/spacy-streamlit/blob/master/spacy_streamlit/__init__.py
+#
+
+from typing import List, Sequence, Tuple, Optional, Dict, Union, Callable
+import streamlit as st
+import spacy
+from spacy.language import Language
+from spacy import displacy
+import pandas as pd
+
+import streamlit as st
+from spacy_streamlit import visualize_spans
+from spacy_streamlit.util import load_model, process_text, get_svg, get_html, LOGO
+
+from pipeline.post_processors import simple_table, const_table, ngrammar, diversity_values
+from skbio import diversity as dv
+
+SPACY_VERSION = tuple(map(int, spacy.__version__.split(".")))
+
+# fmt: off
+# SPAN_ATTRS = ["text", "label_", "start", "end", "start_char", "end_char"]
+SPAN_ATTRS = [
+    "text",
+    "label_",
+    "start",
+    "end",
+]
+
+CATEGORIES = ['ATTRIBUTION', "CITATION", "COUNTER", "DENY", "ENDOPHORIC", "ENTERTAIN", "JUSTIFYING", "MONOGLOSS", "PROCLAIM", "SOURCES"]
+
+def visualize_spans(
+    doc: Union[spacy.tokens.Doc, Dict[str, str]],
+    *,
+    spans_key: str = "sc",
+    attrs: List[str] = SPAN_ATTRS,
+    show_table: bool = True,
+    title: Optional[str] = "Spans",
+    manual: bool = False,
+    displacy_options: Optional[Dict] = None,
+    simple: bool = True,
+):
+    """
+    Visualizer for spans.
+    doc (Doc, Dict): The document to visualize.
+    spans_key (str): Which spans key to render spans from. Default is "sc".
+    attrs (list):  The attributes on the entity Span to be labeled. Attributes are displayed only when the show_table
+    argument is True.
+    show_table (bool): Flag signifying whether to show a table with accompanying span attributes.
+    title (str): The title displayed at the top of the Spans visualization.
+    manual (bool): Flag signifying whether the doc argument is a Doc object or a List of Dicts containing span information.
+    displacy_options (Dict): Dictionary of options to be passed to the displacy render method for generating the HTML to be rendered.
+      See https://spacy.io/api/top-level#displacy_options-span
+    """
+    if SPACY_VERSION < (3, 3, 0):
+        raise ValueError(
+            f"'visualize_spans' requires spacy>=3.3.0. You have spacy=={spacy.__version__}"
+        )
+    if not displacy_options:
+        displacy_options = dict()
+    displacy_options["spans_key"] = spans_key
+
+    if title:
+        st.header(title)
+
+    if manual:
+        if show_table:
+            st.warning(
+                "When the parameter 'manual' is set to True, the parameter 'show_table' must be set to False."
+            )
+        if not isinstance(doc, dict):
+            st.warning(
+                "When the parameter 'manual' is set to True, the parameter 'doc' must be of type 'Dict', not 'spacy.tokens.Doc'."
+            )
+    html = displacy.render(
+        doc,
+        style="span",
+        options=displacy_options,
+        manual=manual,
+    )
+    st.write(f"{get_html(html)}", unsafe_allow_html=True)
+
+    if show_table:
+        # data = [
+        #     [str(getattr(span, attr)) for attr in attrs] + [str(score)]
+        #     for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores'])
+        # ]
+        if simple:
+            data, cols = simple_table(doc, spans_key='sc', attrs=attrs)
+        else:
+            data, cols = const_table(doc, spans_key='sc', attrs=attrs)
+
+        # seq = [s for s in doc.spans[spans_key]]
+
+        if data:
+            df = pd.DataFrame(data, columns=cols)
+            df = df.astype({"start": int, "end": int})
+            df = df.sort_values(by= ['start'])
+            st.subheader("Span information")
+            st.dataframe(
+                df.style.highlight_between(subset='Conf. score', right=.7))
+
+            if not simple:
+                st.subheader("Label counts & Diagnostic confidence score summary")
+                counts = df['label_'].value_counts().reindex(CATEGORIES, fill_value=0)
+
+                print(counts)
+                print(list(counts))
+                label_counts = df.groupby('label_').agg({
+                    "label_":
+                    'count',
+                    "Conf. score": ['median', 'min', 'max']
+                }).round(4).reindex(CATEGORIES, fill_value=0)
+
+                st.dataframe(label_counts)
+                # print(list(label_counts))
+
+                sequences = list(df['label_'])
+                # Engagement ngrams
+                span_bigrams = ngrammar(seq=sequences, n=2, concat=True)
+                span_trigrams = ngrammar(seq=sequences, n=3, concat=True)
+
+                st.dataframe(pd.DataFrame(span_bigrams))
+                st.code(span_trigrams)
+
+
+                st.subheader("Engagement label by grammatical function")
+                label_dep = pd.crosstab(df['grammatical realization'], df['label_'])
+                st.dataframe(label_dep)
+
+                st.subheader('Quantitative results')
+                # st.markdown(
+                #     f"Shannon's index: {dv.alpha.shannon(list(counts), base=2): .3f}")
+                # st.markdown(
+                #     f"Simpson's e index: {1 - dv.alpha.simpson_e(list(counts)): .3f}")
+                
+                div = diversity_values(list(counts))
+                div_data = pd.DataFrame.from_dict(div, orient='index')
+                st.dataframe(div_data)            
+                
+                doc_data = pd.concat([counts, div_data], axis = 0).T
+                filename = "NA"
+                doc_data.insert(0, "filename", filename, True)
+                doc_data.insert(1, "nwords", len(doc), True)
+                st.dataframe(doc_data)            
+            # st.markdown(str(dv.alpha_diversity(metric = "shannon", counts=counts, ids = ['ENTERTAIN', 'ATTRIBUTE', 'CITATION', 'COUNTER', 'DENY', 'ENDORSE', 'PRONOUNCE', 'CONCUR', 'MONOGLOSS', 'SOURCES', 'JUSTIFYING'])))
+            # print(dv.get_alpha_diversity_metrics())