updated model

.gitignore

@@ -1,2 +1,6 @@
 test_run.py
-.DS_Store
+.DS_Store
+analyzer.py
+main.py
+results/*
+inputtexts/*

demo.py

@@ -32,7 +32,7 @@ st.set_page_config(page_title="ENGAGEMENT analyzer (beta ver 0.3)",
 @st.cache(allow_output_mutation=True)
 def load_model():
     # nlp = spacy.load("en_engagement_RoBERTa_context_flz")
-    nlp = spacy.load("en_engagement_spl_RoBERTa_acad")
+    nlp = spacy.load("en_engagement_LSTM")
     return (nlp)
 
 

pipeline/post_processors.py

@@ -19,12 +19,138 @@ def simple_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
     return data, columns
 
 
+# def span_info_aggregator()
+
+def construction_classifier(doc, span):
+    category = span.root.dep_
+    spanroot = span.root
+
+    ##
+    span_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in span]
+    span_dep = [t.dep_ for t in span]
+    span_token = [t.norm_ for t in span]
+    span_tag = [t.tag_ for t in span]
+
+
+    c_dep = [c.dep_ for c in spanroot.children]
+    c_pos = [c.pos_ for c in spanroot.children]
+    c_tag = [c.tag_ for c in spanroot.children]
+
+    ## nesting classifiers
+    if spanroot.dep_ == "conj":
+        while spanroot.dep_ == 'conj':
+            spanroot = spanroot.head
+    if spanroot.dep_ == "poss":
+        while spanroot.dep_ == 'poss':
+            spanroot = spanroot.head
+
+
+    ## Simple classifier
+    if spanroot.dep_ in ['pcomp']:
+        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
+            category = "Gerund"
+
+
+    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj"]:
+        category = "Object"
+    if spanroot.dep_ in ["nsubj", "nsubjpass"]:
+        category = "Subject"
+    if spanroot.dep_ in ["cc"]:
+        category = "Coordinating conjunction"
+
+    if spanroot.dep_ in ["ROOT", "advcl"]:
+        if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
+            category = "It is X that-clause"
+        elif "nsubj" in c_dep and "acomp" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
+            category = "It is X that-clause"
+        elif "nsubj" in c_dep and "oprd" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
+            category = "It is X that-clause"
+        elif "nsubj" in c_dep and "it" in span_token and spanroot.pos_ == "VERB":
+            category = "It VERB that-clause"
+        elif "expl" in c_dep and "NOUN" in c_pos:
+            category = "There is/are NOUN"
+        elif spanroot.pos_ in ["AUX", 'VERB']:
+            category = "Main verb"
+        else:
+            category = spanroot.dep_
+    
+    if spanroot.dep_ in ['attr']:
+        c_head = [c.dep_ for c in spanroot.head.children]
+        if "expl" in c_head and "no_det" in span_t_dep_:
+            category = "There is/are no NOUN"
+
+        
+    # Modal verbs
+    if spanroot.tag_ == "MD":
+        category = "Modal auxiliary"
+    # prep phrases
+    if spanroot.dep_ in ['prep']:
+        category = 'Prepositional Phrase'
+    # adverbial phrases
+    if spanroot.dep_ in ['advmod']:
+        category = "Adverbial modifier"
+        # adverbial phrases
+    if spanroot.dep_ in ['acomp']:
+        category = "Adjectival complement"
+
+    if spanroot.dep_ in ['neg']:
+        category = "Negative particle"
+
+    # Preconjunctions
+    if spanroot.dep_ in ['preconj']:
+        category = "Conjunction"
+
+    # Adverbial clauses
+    ## Check the status of the adverbial clauses carefully
+    if spanroot.dep_ in ['advcl', 'mark', 'acl']:
+        if "mark" in span_dep:
+            category = "Finite adverbial clause"
+        if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"] and "aux" not in c_dep:
+            category = "Non-finite adv clause"
+        # Check whether it has a subject or not
+        # elif "nsubj" in [c.dep_ for c in spanroot.children]:
+        #     category = "Adverbial clauses"
+        # else:
+        #     category = "Other advcl"
+    
+    if spanroot.dep_ in ['relcl', 'ccomp']:
+        head = spanroot.head
+        if ";" in [t.norm_ for t in head.children]:
+            category = "Main verb"
+        elif "nsubj" not in span_dep:
+            category = "Dependent verb"
+
+    if spanroot.dep_ in ['dep']:
+        if spanroot.head.dep_ in ['ROOT', 'ccomp'] and spanroot.head.pos_ in ['AUX', 'VERB'] and spanroot.pos_ in ['AUX', 'VERB']:
+            if spanroot.morph == spanroot.head.morph:
+                category = "Main verb"
+            else:
+                category = "Dependent verb"
+
+
+
+
+    if span.label_ == "CITATION":
+        if "NNP" in span_tag or "NNPS" in span_tag:
+            if span_dep[0] == 'punct' and span_dep[-1] == 'punct':
+                category = "Parenthetical Citation"
+            elif span_tag[0] in ["NNP", "NNPS"]:
+                category = "Narrative Citation"
+        else:
+            category = "Other Citation"
+
+
+    return category
+
+
 def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
                 spans_key: str = "sc",
                 attrs: List[str] = SPAN_ATTRS):
-    columns = attrs + ["Conf. score", 'span dep',
-                       "POS", "POS sequence", "head"]
+    columns = attrs + ["Conf. score", "sent no.", "grammatical realization", 'span dep', "ner", 
+                       "POS", 'span dep seq', "POS sequence", "head", "children", "morphology", ]
     data = []
+    # data = span_info_aggregator(doc, columns)
+    sentences = {s: i for i, s in enumerate(doc.sents)}
 
     for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs['scores']):
 
@@ -32,11 +158,16 @@ def const_table(doc: Union[spacy.tokens.Doc, Dict[str, str]],
         span_info.extend([str(getattr(span, attr)) for attr in attrs])
 
         span_info.append(score)
+        span_info.append(sentences[span.sent])
+        span_info.append(construction_classifier(doc, span))
         span_info.append(span.root.dep_)
+        span_info.append(span.root.ent_type_)
         span_info.append(span.root.tag_)
+        span_info.append("_".join([t.dep_ for t in span]))
         span_info.append("_".join([t.tag_ for t in span]))
         span_info.append(span.root.head.norm_)
-        # span_info.append(span.root.head.dep_ == "ROOT")
+        span_info.append("_".join([c.dep_ for c in span.root.children]))
+        span_info.append(span.root.morph)
         data.append(span_info)
 
     return data, columns

resources/colors.py

@@ -0,0 +1,13 @@
+COLORS_1 = {
+            "ENTERTAIN": "#82b74b",
+            "DENY": '#c94c4c',
+            "COUNTER": "#eea29a",
+            "PRONOUNCE": "#92a8d1",
+            "ENDORSE": "#034f84",
+            "CITATION": "#b2b2b2",
+            "MONOGLOSS": "#3e4444",
+            "ATTRIBUTE": "#f7786b",
+            "ATTRIBUTION": "#f7786b",
+            "PROCLAIM": "#92a8d1"
+        }
+

resources/template_list.py

@@ -0,0 +1,48 @@
+TPL_ENT = """
+<mark class="entity" style="background: {bg}; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;">
+    {text}
+    <span style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem">{label}</span>
+</mark>
+"""
+
+TPL_SPANS = """
+<div class="spans" style="line-height: 4.5;">
+    {text}
+    {span_slices}
+    {span_starts}
+</div>
+"""
+
+TPL_SPAN = """
+<span style="font-weight: bold; display: inline-block; line-height: 3; padding-bottom: 12px;position: relative;">
+    {text}
+    {span_slices}
+    {span_starts}
+</span>
+"""
+
+TPL_SPAN_SLICE = """
+<span style="background: {bg}; top: {top_offset}px;  display: inline-block; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
+</span>
+"""
+
+TPL_SPAN_START = """
+<span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+    <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+       
+        {label}{kb_link}
+    </span>
+</span>
+
+"""
+
+TPL_SPAN_START_RTL = """
+<span style="background: {bg}; top: {top_offset}px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
+    <span style="background: {bg}; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
+        {label}{kb_link}
+    </span>
+</span>
+"""
+
+DEFAULT_TEXT = """Tickner said regardless of the result, the royal commission was a waste of money and he would proceed with a separate inquiry into the issue headed by Justice Jane Matthews. His attack came as the Aboriginal women involved in the case demanded a female minister examine the religious beliefs they claim are inherent in their fight against a bridge to the island near Goolwa in South Australia."""
+

utils/util.py

@@ -8,6 +8,7 @@ def preprocess(text):
     text = re.sub('\n', ' ', text)
     text = re.sub('\s+', " ", text)
     text = re.sub('&&&&&&&&#&#&#&#&', '\n\n', text)
+    text = re.sub("--- Para SEP ---", '\n', text)
     return text